From 792d5599128300ac582c626e4a77ce61cfed2e23 Mon Sep 17 00:00:00 2001 From: nseratt Date: Mon, 18 Nov 2019 16:19:45 -0500 Subject: [PATCH 01/20] added arg for regexignore to exclude results where the html body matches a provided regex pattern example usage: --regexignore "(Page not found|error)" --- args.go | 11 ++++++++++- main.go | 8 ++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/args.go b/args.go index 90cca52..e09d060 100644 --- a/args.go +++ b/args.go @@ -54,6 +54,7 @@ type config struct { paths string hosts string output string + regexignore string noHeaders bool requester requester @@ -115,6 +116,12 @@ func processArgs() config { flag.BoolVar(&verbose, "verbose", false, "") flag.BoolVar(&verbose, "v", false, "") + regexignore := "" + flag.StringVar(®exignore, "regexignore", "", "") + flag.StringVar(®exignore, "x", "", "") + + + flag.Parse() // paths might be in a file, or it might be a single value @@ -156,6 +163,7 @@ func processArgs() config { hosts: hosts, output: output, noHeaders: noHeaders, + regexignore: regexignore, } } @@ -176,7 +184,8 @@ func init() { h += " -s, --savestatus <status> Save only responses with specific status code\n" h += " -t, --timeout <millis> Set the HTTP timeout (default: 10000)\n" h += " -v, --verbose Verbose mode\n" - h += " -X, --method <method> HTTP method (default: GET)\n\n" + h += " -X, --method <method> HTTP method (default: GET)\n" + h += " -x, --regexignore <string> Ignore results where the body matches a regex pattern\n\n" h += "Defaults:\n" h += " pathsFile: ./paths\n" diff --git a/main.go b/main.go index f2a73a2..3e14e52 100644 --- a/main.go +++ b/main.go @@ -93,6 +93,14 @@ func main() { continue } + if c.regexignore != "" { + matched, _ := regexp.MatchString(c.regexignore,res.String()) + + if matched { + continue + } + } + path, err := res.save(c.output, c.noHeaders) if err != nil { fmt.Fprintf(os.Stderr, "failed to save file: %s\n", err) From 7a2bd8a02d61f543b9bae8e5d9c5e6e712e2d6d8 Mon Sep 17 00:00:00 2001 From: nseratt <nseratt@gmail.com> Date: Mon, 18 Nov 2019 16:30:01 -0500 Subject: [PATCH 02/20] Update main.go --- main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/main.go b/main.go index 3e14e52..7b097a7 100644 --- a/main.go +++ b/main.go @@ -8,6 +8,7 @@ import ( "path/filepath" "sync" "time" + "regexp" ) const ( From d06f0bec570fe0e596e645544154ceff5fd03086 Mon Sep 17 00:00:00 2001 From: Gwyn Hannay <gwyn@hannay.id.au> Date: Fri, 3 Jul 2020 01:26:43 +0000 Subject: [PATCH 03/20] Adds sr flag for saving responses with specific string --- README.mkd | 10 ++++++++++ args.go | 28 ++++++++++++++++++---------- main.go | 5 +++++ 3 files changed, 33 insertions(+), 10 deletions(-) mode change 100644 => 100755 args.go mode change 100644 => 100755 main.go diff --git a/README.mkd b/README.mkd index 0a8bf16..5f182b3 100644 --- a/README.mkd +++ b/README.mkd @@ -140,6 +140,7 @@ Options: -H, --header <header> Send a custom HTTP header -r, --rawhttp Use the rawhttp library for requests (experimental) -s, --savestatus <status> Save only responses with specific status code + -sr, --saveresp <string> Save only responses containing specific string -v, --verbose Verbose mode -X, --method <method> HTTP method (default: GET) @@ -233,6 +234,15 @@ use the `-s` or `--savestatus` option: ▶ meg --savestatus 200 /robots.txt ``` +### Saving Only Certain Response Strings + +If you only want to save results that contained a certain string, you can +use the `-sr` or `--saveresp` option: + +``` +▶ meg --saveresp '": "v1"' /robots.txt +``` + ### Specifying The Method You can specify which HTTP method to use with the `-X` or `--method` option: diff --git a/args.go b/args.go old mode 100644 new mode 100755 index 90cca52..b0b633d --- a/args.go +++ b/args.go @@ -48,6 +48,7 @@ type config struct { followLocation bool method string saveStatus saveStatusArgs + saveResp string timeout int verbose bool @@ -96,6 +97,11 @@ func processArgs() config { flag.Var(&saveStatus, "savestatus", "") flag.Var(&saveStatus, "s", "") + // saveResp params + saveResp := "" + flag.StringVar(&saveResp, "saveResp", "", "") + flag.StringVar(&saveResp, "sr", "", "") + // timeout param timeout := 10000 flag.IntVar(&timeout, "timeout", 10000, "") @@ -149,6 +155,7 @@ func processArgs() config { followLocation: followLocation, method: method, saveStatus: saveStatus, + saveResp: saveResp, timeout: timeout, requester: requesterFn, verbose: verbose, @@ -167,16 +174,17 @@ func init() { h += " meg [path|pathsFile] [hostsFile] [outputDir]\n\n" h += "Options:\n" - h += " -b, --body <val> Set the request body\n" - h += " -c, --concurrency <val> Set the concurrency level (default: 20)\n" - h += " -d, --delay <millis> Milliseconds between requests to the same host (default: 5000)\n" - h += " -H, --header <header> Send a custom HTTP header\n" - h += " -L, --location Follow redirects / location header\n" - h += " -r, --rawhttp Use the rawhttp library for requests (experimental)\n" - h += " -s, --savestatus <status> Save only responses with specific status code\n" - h += " -t, --timeout <millis> Set the HTTP timeout (default: 10000)\n" - h += " -v, --verbose Verbose mode\n" - h += " -X, --method <method> HTTP method (default: GET)\n\n" + h += " -b, --body <val> Set the request body\n" + h += " -c, --concurrency <val> Set the concurrency level (default: 20)\n" + h += " -d, --delay <millis> Milliseconds between requests to the same host (default: 5000)\n" + h += " -H, --header <header> Send a custom HTTP header\n" + h += " -L, --location Follow redirects / location header\n" + h += " -r, --rawhttp Use the rawhttp library for requests (experimental)\n" + h += " -s, --savestatus <status> Save only responses with specific status code\n" + h += " -sr, --saveresp <string> Save only responses containing specific string\n" + h += " -t, --timeout <millis> Set the HTTP timeout (default: 10000)\n" + h += " -v, --verbose Verbose mode\n" + h += " -X, --method <method> HTTP method (default: GET)\n\n" h += "Defaults:\n" h += " pathsFile: ./paths\n" diff --git a/main.go b/main.go old mode 100644 new mode 100755 index f2a73a2..12542da --- a/main.go +++ b/main.go @@ -6,6 +6,7 @@ import ( "net/url" "os" "path/filepath" + "strings" "sync" "time" ) @@ -88,6 +89,10 @@ func main() { continue } + if len(c.saveResp) > 0 && !(strings.Contains(strings.Join(res.headers, ""), c.saveResp) || (strings.Contains(string(res.body), c.saveResp))) { + continue + } + if res.err != nil { fmt.Fprintf(os.Stderr, "request failed: %s\n", res.err) continue From 70a2c3c3783cb056e31e0a05af80b4a3dfe05661 Mon Sep 17 00:00:00 2001 From: Gwyn Hannay <gwyn@hannay.id.au> Date: Sun, 5 Jul 2020 00:43:58 +0000 Subject: [PATCH 04/20] Adds dr flag for discarding responses with specific string --- README.mkd | 10 ++++++++++ args.go | 12 ++++++++++-- main.go | 4 ++++ 3 files changed, 24 insertions(+), 2 deletions(-) mode change 100644 => 100755 README.mkd diff --git a/README.mkd b/README.mkd old mode 100644 new mode 100755 index 5f182b3..23e39b4 --- a/README.mkd +++ b/README.mkd @@ -137,6 +137,7 @@ Usage: Options: -c, --concurrency <val> Set the concurrency level (defaut: 20) -d, --delay <val> Milliseconds between requests to the same host (default: 5000) + -dr, --discresp <string> Discard responses containing specific string -H, --header <header> Send a custom HTTP header -r, --rawhttp Use the rawhttp library for requests (experimental) -s, --savestatus <status> Save only responses with specific status code @@ -243,6 +244,15 @@ use the `-sr` or `--saveresp` option: ▶ meg --saveresp '": "v1"' /robots.txt ``` +### Discarding Certain Response Strings + +Alternatively, if you don't want to save results that contained a certain string, you can +use the `-dr` or `--discresp` option: + +``` +▶ meg --discresp '": "v1"' /robots.txt +``` + ### Specifying The Method You can specify which HTTP method to use with the `-X` or `--method` option: diff --git a/args.go b/args.go index b0b633d..efe7e7a 100755 --- a/args.go +++ b/args.go @@ -44,6 +44,7 @@ type config struct { body string concurrency int delay int + discResp string headers headerArgs followLocation bool method string @@ -77,6 +78,11 @@ func processArgs() config { flag.IntVar(&delay, "delay", 5000, "") flag.IntVar(&delay, "d", 5000, "") + // discResp params + discResp := "" + flag.StringVar(&discResp, "discresp", "", "") + flag.StringVar(&discResp, "dr", "", "") + // headers params var headers headerArgs flag.Var(&headers, "header", "") @@ -99,7 +105,7 @@ func processArgs() config { // saveResp params saveResp := "" - flag.StringVar(&saveResp, "saveResp", "", "") + flag.StringVar(&saveResp, "saveresp", "", "") flag.StringVar(&saveResp, "sr", "", "") // timeout param @@ -151,6 +157,7 @@ func processArgs() config { body: body, concurrency: concurrency, delay: delay, + discResp: discResp, headers: headers, followLocation: followLocation, method: method, @@ -177,11 +184,12 @@ func init() { h += " -b, --body <val> Set the request body\n" h += " -c, --concurrency <val> Set the concurrency level (default: 20)\n" h += " -d, --delay <millis> Milliseconds between requests to the same host (default: 5000)\n" + h += " -dr, --discresp <string> Discard responses containing specific string\n" h += " -H, --header <header> Send a custom HTTP header\n" h += " -L, --location Follow redirects / location header\n" h += " -r, --rawhttp Use the rawhttp library for requests (experimental)\n" h += " -s, --savestatus <status> Save only responses with specific status code\n" - h += " -sr, --saveresp <string> Save only responses containing specific string\n" + h += " -sr, --saveresp <string> Save only responses containing specific string\n" h += " -t, --timeout <millis> Set the HTTP timeout (default: 10000)\n" h += " -v, --verbose Verbose mode\n" h += " -X, --method <method> HTTP method (default: GET)\n\n" diff --git a/main.go b/main.go index 12542da..488b305 100755 --- a/main.go +++ b/main.go @@ -93,6 +93,10 @@ func main() { continue } + if len(c.discResp) > 0 && (strings.Contains(strings.Join(res.headers, ""), c.discResp) || (strings.Contains(string(res.body), c.discResp))) { + continue + } + if res.err != nil { fmt.Fprintf(os.Stderr, "request failed: %s\n", res.err) continue From a766b34295164c3ebb18b3083b3a101746e2edec Mon Sep 17 00:00:00 2001 From: Gwyn <gwyn@hannay.id.au> Date: Thu, 20 Aug 2020 13:29:14 +0800 Subject: [PATCH 05/20] create regexkeep with ignore --- args.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/args.go b/args.go index 85f757d..fdf8e46 100644 --- a/args.go +++ b/args.go @@ -48,6 +48,8 @@ type config struct { headers headerArgs followLocation bool method string + regexignore string + regexkeep string saveStatus saveStatusArgs saveResp string timeout int @@ -56,7 +58,6 @@ type config struct { paths string hosts string output string - regexignore string noHeaders bool requester requester @@ -130,7 +131,7 @@ func processArgs() config { regexignore := "" flag.StringVar(®exignore, "regexignore", "", "") - flag.StringVar(®exignore, "x", "", "") + flag.StringVar(®exignore, "ri", "", "") @@ -168,6 +169,8 @@ func processArgs() config { headers: headers, followLocation: followLocation, method: method, + regexignore: regexignore, + regexkeep: regexkeep, saveStatus: saveStatus, saveResp: saveResp, timeout: timeout, @@ -177,7 +180,6 @@ func processArgs() config { hosts: hosts, output: output, noHeaders: noHeaders, - regexignore: regexignore, } } @@ -196,6 +198,8 @@ func init() { h += " -H, --header <header> Send a custom HTTP header\n" h += " -L, --location Follow redirects / location header\n" h += " -r, --rawhttp Use the rawhttp library for requests (experimental)\n" + h += " -ri, --regexignore <string> Ignore results where the body matches a regex pattern\n\n" + h += " -rk, --regexkeep <string> Keep results where the body matches a regex pattern\n\n" h += " -s, --savestatus <status> Save only responses with specific status code\n" h += " -sr, --saveresp <string> Save only responses containing specific string\n" h += " -t, --timeout <millis> Set the HTTP timeout (default: 10000)\n" From a6908091edba3fa0e3517878131ce411052625c2 Mon Sep 17 00:00:00 2001 From: Gwyn <gwyn@hannay.id.au> Date: Thu, 20 Aug 2020 13:32:58 +0800 Subject: [PATCH 06/20] setup regex in main --- main.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index f55c05b..3381e36 100755 --- a/main.go +++ b/main.go @@ -103,10 +103,18 @@ func main() { continue } - if c.regexignore != "" { + if len(c.regexignore) > 0 { matched, _ := regexp.MatchString(c.regexignore,res.String()) - if matched { + if matched { + continue + } + } + + if len(c.regexkeep) > 0 { + matched, _ := regexp.MatchString(c.regexkeep,res.String()) + + if not matched { continue } } From 92e9a0b906f6b838adc343e8dc615a9892d11dd5 Mon Sep 17 00:00:00 2001 From: Gwyn <gwyn@hannay.id.au> Date: Thu, 20 Aug 2020 13:36:15 +0800 Subject: [PATCH 07/20] syntax for no match --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index 3381e36..96a7434 100755 --- a/main.go +++ b/main.go @@ -114,7 +114,7 @@ func main() { if len(c.regexkeep) > 0 { matched, _ := regexp.MatchString(c.regexkeep,res.String()) - if not matched { + if !matched { continue } } From dd6299a2f54b99f419436096d5ee545d129c7de9 Mon Sep 17 00:00:00 2001 From: Gwyn <gwyn@hannay.id.au> Date: Thu, 20 Aug 2020 13:38:39 +0800 Subject: [PATCH 08/20] define regexkeep --- args.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/args.go b/args.go index fdf8e46..aa51c01 100644 --- a/args.go +++ b/args.go @@ -100,6 +100,14 @@ func processArgs() config { flag.StringVar(&method, "method", "GET", "") flag.StringVar(&method, "X", "GET", "") + regexignore := "" + flag.StringVar(®exignore, "regexignore", "", "") + flag.StringVar(®exignore, "ri", "", "") + + regexkeep := "" + flag.StringVar(®exkeep, "regexkeep", "", "") + flag.StringVar(®exkeep, "rk", "", "") + // savestatus params var saveStatus saveStatusArgs flag.Var(&saveStatus, "savestatus", "") @@ -129,10 +137,6 @@ func processArgs() config { flag.BoolVar(&verbose, "verbose", false, "") flag.BoolVar(&verbose, "v", false, "") - regexignore := "" - flag.StringVar(®exignore, "regexignore", "", "") - flag.StringVar(®exignore, "ri", "", "") - flag.Parse() From e2d81cac6137235be936d966b8cee9239cb65334 Mon Sep 17 00:00:00 2001 From: Gwyn <gwyn@hannay.id.au> Date: Thu, 20 Aug 2020 13:48:35 +0800 Subject: [PATCH 09/20] parameter change --- args.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/args.go b/args.go index aa51c01..2caa41c 100644 --- a/args.go +++ b/args.go @@ -105,8 +105,8 @@ func processArgs() config { flag.StringVar(®exignore, "ri", "", "") regexkeep := "" - flag.StringVar(®exkeep, "regexkeep", "", "") - flag.StringVar(®exkeep, "rk", "", "") + flag.StringVar(®exkeep, "regexkeep", "") + flag.StringVar(®exkeep, "rk", "") // savestatus params var saveStatus saveStatusArgs From a4bd7a9b2908c5550536f5734efcdffdb8147c44 Mon Sep 17 00:00:00 2001 From: Gwyn <gwyn@hannay.id.au> Date: Thu, 20 Aug 2020 13:50:03 +0800 Subject: [PATCH 10/20] undo --- args.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/args.go b/args.go index 2caa41c..aa51c01 100644 --- a/args.go +++ b/args.go @@ -105,8 +105,8 @@ func processArgs() config { flag.StringVar(®exignore, "ri", "", "") regexkeep := "" - flag.StringVar(®exkeep, "regexkeep", "") - flag.StringVar(®exkeep, "rk", "") + flag.StringVar(®exkeep, "regexkeep", "", "") + flag.StringVar(®exkeep, "rk", "", "") // savestatus params var saveStatus saveStatusArgs From 3a160da8ad5ad52e12f7f2153c5110be2536e010 Mon Sep 17 00:00:00 2001 From: Gwyn <gwyn@hannay.id.au> Date: Thu, 20 Aug 2020 14:02:47 +0800 Subject: [PATCH 11/20] fix up docs --- README.mkd | 20 +++++++++++--------- args.go | 39 ++++++++++++++++++++------------------- main.go | 32 ++++++++++++++++---------------- 3 files changed, 47 insertions(+), 44 deletions(-) diff --git a/README.mkd b/README.mkd index 23e39b4..d33998d 100755 --- a/README.mkd +++ b/README.mkd @@ -135,15 +135,17 @@ Usage: meg [options] [path|pathsFile] [hostsFile] [outputDir] Options: - -c, --concurrency <val> Set the concurrency level (defaut: 20) - -d, --delay <val> Milliseconds between requests to the same host (default: 5000) - -dr, --discresp <string> Discard responses containing specific string - -H, --header <header> Send a custom HTTP header - -r, --rawhttp Use the rawhttp library for requests (experimental) - -s, --savestatus <status> Save only responses with specific status code - -sr, --saveresp <string> Save only responses containing specific string - -v, --verbose Verbose mode - -X, --method <method> HTTP method (default: GET) + -c, --concurrency <val> Set the concurrency level (defaut: 20) + -d, --delay <val> Milliseconds between requests to the same host (default: 5000) + -dr, --discresp <string> Discard responses containing specific string + -H, --header <header> Send a custom HTTP header + -r, --rawhttp Use the rawhttp library for requests (experimental) + -ri, --regexignore <string> Ignore results where the body matches a regex pattern + -rk, --regexkeep <string> Keep results where the body matches a regex pattern + -sr, --saveresp <string> Save only responses containing specific string + -s, --savestatus <status> Save only responses with specific status code + -v, --verbose Verbose mode + -X, --method <method> HTTP method (default: GET) Defaults: pathsFile: ./paths diff --git a/args.go b/args.go index aa51c01..833f270 100644 --- a/args.go +++ b/args.go @@ -48,10 +48,10 @@ type config struct { headers headerArgs followLocation bool method string - regexignore string - regexkeep string - saveStatus saveStatusArgs + regexIgnore string + regexKeep string saveResp string + saveStatus saveStatusArgs timeout int verbose bool @@ -100,24 +100,26 @@ func processArgs() config { flag.StringVar(&method, "method", "GET", "") flag.StringVar(&method, "X", "GET", "") - regexignore := "" - flag.StringVar(®exignore, "regexignore", "", "") - flag.StringVar(®exignore, "ri", "", "") + // regexIgnore params + regexIgnore := "" + flag.StringVar(®exIgnore, "regexIgnore", "", "") + flag.StringVar(®exIgnore, "ri", "", "") - regexkeep := "" - flag.StringVar(®exkeep, "regexkeep", "", "") - flag.StringVar(®exkeep, "rk", "", "") - - // savestatus params - var saveStatus saveStatusArgs - flag.Var(&saveStatus, "savestatus", "") - flag.Var(&saveStatus, "s", "") + // regexKeep params + regexKeep := "" + flag.StringVar(®exKeep, "regexKeep", "", "") + flag.StringVar(®exKeep, "rk", "", "") // saveResp params saveResp := "" flag.StringVar(&saveResp, "saveresp", "", "") flag.StringVar(&saveResp, "sr", "", "") + // savestatus params + var saveStatus saveStatusArgs + flag.Var(&saveStatus, "savestatus", "") + flag.Var(&saveStatus, "s", "") + // timeout param timeout := 10000 flag.IntVar(&timeout, "timeout", 10000, "") @@ -173,10 +175,10 @@ func processArgs() config { headers: headers, followLocation: followLocation, method: method, - regexignore: regexignore, - regexkeep: regexkeep, - saveStatus: saveStatus, + regexIgnore: regexIgnore, + regexKeep: regexKeep, saveResp: saveResp, + saveStatus: saveStatus, timeout: timeout, requester: requesterFn, verbose: verbose, @@ -204,12 +206,11 @@ func init() { h += " -r, --rawhttp Use the rawhttp library for requests (experimental)\n" h += " -ri, --regexignore <string> Ignore results where the body matches a regex pattern\n\n" h += " -rk, --regexkeep <string> Keep results where the body matches a regex pattern\n\n" - h += " -s, --savestatus <status> Save only responses with specific status code\n" h += " -sr, --saveresp <string> Save only responses containing specific string\n" + h += " -s, --savestatus <status> Save only responses with specific status code\n" h += " -t, --timeout <millis> Set the HTTP timeout (default: 10000)\n" h += " -v, --verbose Verbose mode\n" h += " -X, --method <method> HTTP method (default: GET)\n\n" - h += " -x, --regexignore <string> Ignore results where the body matches a regex pattern\n\n" h += "Defaults:\n" h += " pathsFile: ./paths\n" diff --git a/main.go b/main.go index 96a7434..7e03c54 100755 --- a/main.go +++ b/main.go @@ -102,22 +102,6 @@ func main() { fmt.Fprintf(os.Stderr, "request failed: %s\n", res.err) continue } - - if len(c.regexignore) > 0 { - matched, _ := regexp.MatchString(c.regexignore,res.String()) - - if matched { - continue - } - } - - if len(c.regexkeep) > 0 { - matched, _ := regexp.MatchString(c.regexkeep,res.String()) - - if !matched { - continue - } - } path, err := res.save(c.output, c.noHeaders) if err != nil { @@ -129,6 +113,22 @@ func main() { if c.verbose { fmt.Printf("%s", line) } + + if len(c.regexIgnore) > 0 { + matched, _ := regexp.MatchString(c.regexIgnore,res.String()) + + if matched { + continue + } + } + + if len(c.regexKeep) > 0 { + matched, _ := regexp.MatchString(c.regexKeep,res.String()) + + if !matched { + continue + } + } } owg.Done() }() From 92ca7f7a380d3835cfea833040389fca58b4b4bf Mon Sep 17 00:00:00 2001 From: GwynHannay <gwyn@hannay.id.au> Date: Thu, 20 Aug 2020 06:06:00 +0000 Subject: [PATCH 12/20] formatting --- args.go | 2 -- main.go | 8 ++++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/args.go b/args.go index 833f270..9551c07 100644 --- a/args.go +++ b/args.go @@ -139,8 +139,6 @@ func processArgs() config { flag.BoolVar(&verbose, "verbose", false, "") flag.BoolVar(&verbose, "v", false, "") - - flag.Parse() // paths might be in a file, or it might be a single value diff --git a/main.go b/main.go index 7e03c54..4071216 100755 --- a/main.go +++ b/main.go @@ -6,10 +6,10 @@ import ( "net/url" "os" "path/filepath" + "regexp" "strings" "sync" "time" - "regexp" ) const ( @@ -102,7 +102,7 @@ func main() { fmt.Fprintf(os.Stderr, "request failed: %s\n", res.err) continue } - + path, err := res.save(c.output, c.noHeaders) if err != nil { fmt.Fprintf(os.Stderr, "failed to save file: %s\n", err) @@ -115,7 +115,7 @@ func main() { } if len(c.regexIgnore) > 0 { - matched, _ := regexp.MatchString(c.regexIgnore,res.String()) + matched, _ := regexp.MatchString(c.regexIgnore, res.String()) if matched { continue @@ -123,7 +123,7 @@ func main() { } if len(c.regexKeep) > 0 { - matched, _ := regexp.MatchString(c.regexKeep,res.String()) + matched, _ := regexp.MatchString(c.regexKeep, res.String()) if !matched { continue From a0b54ab95a19d85fbc87f368094cc537314f3a7e Mon Sep 17 00:00:00 2001 From: swarley7 <swarley7@users.noreply.github.com> Date: Mon, 2 Aug 2021 14:41:56 +0800 Subject: [PATCH 13/20] fixed regex stuff --- main.go | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/main.go b/main.go index 4071216..82aa197 100755 --- a/main.go +++ b/main.go @@ -97,6 +97,21 @@ func main() { if len(c.discResp) > 0 && (strings.Contains(strings.Join(res.headers, ""), c.discResp) || (strings.Contains(string(res.body), c.discResp))) { continue } + if len(c.regexIgnore) > 0 { + re := regexp.MustCompile(c.regexIgnore) + matched := re.MatchString(res.String()) + if matched { + continue + } + } + + if len(c.regexKeep) > 0 { + re := regexp.MustCompile(c.regexKeep) + matched := re.MatchString(res.String()) + if !matched { + continue + } + } if res.err != nil { fmt.Fprintf(os.Stderr, "request failed: %s\n", res.err) @@ -113,22 +128,6 @@ func main() { if c.verbose { fmt.Printf("%s", line) } - - if len(c.regexIgnore) > 0 { - matched, _ := regexp.MatchString(c.regexIgnore, res.String()) - - if matched { - continue - } - } - - if len(c.regexKeep) > 0 { - matched, _ := regexp.MatchString(c.regexKeep, res.String()) - - if !matched { - continue - } - } } owg.Done() }() From c71ea4ff24812bf7d8c0cc0518fa04b0d440963e Mon Sep 17 00:00:00 2001 From: Gwyn <gwyn@hannay.id.au> Date: Mon, 27 Dec 2021 08:20:19 +0800 Subject: [PATCH 14/20] Update README.mkd --- README.mkd | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.mkd b/README.mkd index d33998d..bf333e3 100755 --- a/README.mkd +++ b/README.mkd @@ -5,7 +5,7 @@ meg is a tool for fetching lots of URLs but still being 'nice' to servers. It can be used to fetch many paths for many hosts; fetching one path for all hosts before moving on to the next path and repeating. -You get lots of results quickly, but non of the individual hosts get +You get lots of results quickly, but none of the individual hosts get flooded with traffic. ## Install @@ -14,10 +14,10 @@ meg is written in Go and has no run-time dependencies. If you have Go 1.9 or later installed and configured you can install meg with `go get`: ``` -▶ go get -u github.com/tomnomnom/meg +▶ go get -u github.com/GwynHannay/meg ``` -Or [download a binary](https://github.com/tomnomnom/meg/releases) and +Or [download a binary](https://github.com/GwynHannay/meg/releases) and put it somewhere in your `$PATH` (e.g. in `/usr/bin/`). ### Install Errors @@ -25,12 +25,12 @@ put it somewhere in your `$PATH` (e.g. in `/usr/bin/`). If you see an error like this it means your version of Go is too old: ``` -# github.com/tomnomnom/rawhttp -/root/go/src/github.com/tomnomnom/rawhttp/request.go:102: u.Hostname undefined ( +# github.com/GwynHannay/rawhttp +/root/go/src/github.com/GwynHannay/rawhttp/request.go:102: u.Hostname undefined ( type *url.URL has no field or method Hostname) -/root/go/src/github.com/tomnomnom/rawhttp/request.go:103: u.Port undefined (type +/root/go/src/github.com/GwynHannay/rawhttp/request.go:103: u.Port undefined (type *url.URL has no field or method Port) - /root/go/src/github.com/tomnomnom/rawhttp/request.go:259: undefined: x509.System + /root/go/src/github.com/GwynHannay/rawhttp/request.go:259: undefined: x509.System CertPool ``` @@ -216,7 +216,7 @@ the Go HTTP client will fail: request failed: parse https://example.org/%%0a0afoo:bar: invalid URL escape "%%0" ``` -You can use the `-r` or `--rawhttp` flag to enable use of the [rawhttp](https://github.com/tomnomnom/rawhttp) +You can use the `-r` or `--rawhttp` flag to enable use of the [rawhttp](https://github.com/GwynHannay/rawhttp) library, which does little to no validation on the request: ``` From 0cae34baaacb4521061bc0d45df5864836517fac Mon Sep 17 00:00:00 2001 From: Gwyn <gwyn@hannay.id.au> Date: Mon, 27 Dec 2021 08:28:28 +0800 Subject: [PATCH 15/20] Create main.yml --- .github/workflows/main.yml | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .github/workflows/main.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..6e887ea --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,37 @@ +# This is a basic workflow to help you get started with Actions + +name: CI + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the master branch + push: + branches: [ master ] + pull_request: + branches: [ master ] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + build: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + + # Runs a single command using the runners shell + - name: Go Release Binaries + uses: wangyoucao577/go-release-action@v1.22 + + + # Runs a set of commands using the runners shell + - name: Run a multi-line script + run: | + echo Add other actions to build, + echo test, and deploy your project. From 8e45bb16079b14c1d1ae18e6ba5b8395502410f3 Mon Sep 17 00:00:00 2001 From: Gwyn <gwyn@hannay.id.au> Date: Mon, 27 Dec 2021 08:40:45 +0800 Subject: [PATCH 16/20] Update README.mkd --- README.mkd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.mkd b/README.mkd index bf333e3..7fcc648 100755 --- a/README.mkd +++ b/README.mkd @@ -216,7 +216,7 @@ the Go HTTP client will fail: request failed: parse https://example.org/%%0a0afoo:bar: invalid URL escape "%%0" ``` -You can use the `-r` or `--rawhttp` flag to enable use of the [rawhttp](https://github.com/GwynHannay/rawhttp) +You can use the `-r` or `--rawhttp` flag to enable use of the [rawhttp](https://github.com/tomnomnom/rawhttp) library, which does little to no validation on the request: ``` From 420c3525f27b75bf76094fa67a3d30c74536e0e8 Mon Sep 17 00:00:00 2001 From: Gwyn <gwyn@hannay.id.au> Date: Mon, 27 Dec 2021 08:41:55 +0800 Subject: [PATCH 17/20] Update main.yml --- .github/workflows/main.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6e887ea..5571788 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -25,10 +25,6 @@ jobs: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v2 - # Runs a single command using the runners shell - - name: Go Release Binaries - uses: wangyoucao577/go-release-action@v1.22 - # Runs a set of commands using the runners shell - name: Run a multi-line script From 1ebc417e708316932080a8b5381ba18e52e71742 Mon Sep 17 00:00:00 2001 From: Gwyn <gwyn@hannay.id.au> Date: Mon, 27 Dec 2021 08:42:25 +0800 Subject: [PATCH 18/20] Update README.mkd --- README.mkd | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.mkd b/README.mkd index 7fcc648..c32805a 100755 --- a/README.mkd +++ b/README.mkd @@ -17,9 +17,6 @@ or later installed and configured you can install meg with `go get`: ▶ go get -u github.com/GwynHannay/meg ``` -Or [download a binary](https://github.com/GwynHannay/meg/releases) and -put it somewhere in your `$PATH` (e.g. in `/usr/bin/`). - ### Install Errors If you see an error like this it means your version of Go is too old: From 26416cd1c519bc528e4ed57e7dda3da9f2abd643 Mon Sep 17 00:00:00 2001 From: Gwyn <gwyn@hannay.id.au> Date: Mon, 27 Dec 2021 08:43:47 +0800 Subject: [PATCH 19/20] Update README.mkd --- README.mkd | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.mkd b/README.mkd index c32805a..260483f 100755 --- a/README.mkd +++ b/README.mkd @@ -22,12 +22,12 @@ or later installed and configured you can install meg with `go get`: If you see an error like this it means your version of Go is too old: ``` -# github.com/GwynHannay/rawhttp -/root/go/src/github.com/GwynHannay/rawhttp/request.go:102: u.Hostname undefined ( +# github.com/tomnomnom/rawhttp +/root/go/src/github.com/tomnomnom/rawhttp/request.go:102: u.Hostname undefined ( type *url.URL has no field or method Hostname) -/root/go/src/github.com/GwynHannay/rawhttp/request.go:103: u.Port undefined (type +/root/go/src/github.com/tomnomnom/rawhttp/request.go:103: u.Port undefined (type *url.URL has no field or method Port) - /root/go/src/github.com/GwynHannay/rawhttp/request.go:259: undefined: x509.System + /root/go/src/github.com/tomnomnom/rawhttp/request.go:259: undefined: x509.System CertPool ``` From be41c71e9369eb84d330cb2d813ab32f34e95248 Mon Sep 17 00:00:00 2001 From: Gwyn <gwyn@hannay.id.au> Date: Tue, 4 Jan 2022 17:10:57 +0800 Subject: [PATCH 20/20] Update README.mkd --- README.mkd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.mkd b/README.mkd index 260483f..d81b55e 100755 --- a/README.mkd +++ b/README.mkd @@ -1,6 +1,6 @@ # meg -meg is a tool for fetching lots of URLs but still being 'nice' to servers. +Created by [Tom Hudson](https://github.com/tomnomnom) (original repo [here](https://github.com/tomnomnom/meg)), meg is a tool for fetching lots of URLs but still being 'nice' to servers. It can be used to fetch many paths for many hosts; fetching one path for all hosts before moving on to the next path and repeating.