diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..5571788 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,33 @@ +# This is a basic workflow to help you get started with Actions + +name: CI + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the master branch + push: + branches: [ master ] + pull_request: + branches: [ master ] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + build: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + + + # Runs a set of commands using the runners shell + - name: Run a multi-line script + run: | + echo Add other actions to build, + echo test, and deploy your project. diff --git a/README.mkd b/README.mkd old mode 100644 new mode 100755 index 0a8bf16..d81b55e --- a/README.mkd +++ b/README.mkd @@ -1,11 +1,11 @@ # meg -meg is a tool for fetching lots of URLs but still being 'nice' to servers. +Created by [Tom Hudson](https://github.com/tomnomnom) (original repo [here](https://github.com/tomnomnom/meg)), meg is a tool for fetching lots of URLs but still being 'nice' to servers. It can be used to fetch many paths for many hosts; fetching one path for all hosts before moving on to the next path and repeating. -You get lots of results quickly, but non of the individual hosts get +You get lots of results quickly, but none of the individual hosts get flooded with traffic. ## Install @@ -14,12 +14,9 @@ meg is written in Go and has no run-time dependencies. If you have Go 1.9 or later installed and configured you can install meg with `go get`: ``` -▶ go get -u github.com/tomnomnom/meg +▶ go get -u github.com/GwynHannay/meg ``` -Or [download a binary](https://github.com/tomnomnom/meg/releases) and -put it somewhere in your `$PATH` (e.g. in `/usr/bin/`). - ### Install Errors If you see an error like this it means your version of Go is too old: @@ -135,13 +132,17 @@ Usage: meg [options] [path|pathsFile] [hostsFile] [outputDir] Options: - -c, --concurrency Set the concurrency level (defaut: 20) - -d, --delay Milliseconds between requests to the same host (default: 5000) - -H, --header
Send a custom HTTP header - -r, --rawhttp Use the rawhttp library for requests (experimental) - -s, --savestatus Save only responses with specific status code - -v, --verbose Verbose mode - -X, --method HTTP method (default: GET) + -c, --concurrency Set the concurrency level (defaut: 20) + -d, --delay Milliseconds between requests to the same host (default: 5000) + -dr, --discresp Discard responses containing specific string + -H, --header
Send a custom HTTP header + -r, --rawhttp Use the rawhttp library for requests (experimental) + -ri, --regexignore Ignore results where the body matches a regex pattern + -rk, --regexkeep Keep results where the body matches a regex pattern + -sr, --saveresp Save only responses containing specific string + -s, --savestatus Save only responses with specific status code + -v, --verbose Verbose mode + -X, --method HTTP method (default: GET) Defaults: pathsFile: ./paths @@ -233,6 +234,24 @@ use the `-s` or `--savestatus` option: ▶ meg --savestatus 200 /robots.txt ``` +### Saving Only Certain Response Strings + +If you only want to save results that contained a certain string, you can +use the `-sr` or `--saveresp` option: + +``` +▶ meg --saveresp '": "v1"' /robots.txt +``` + +### Discarding Certain Response Strings + +Alternatively, if you don't want to save results that contained a certain string, you can +use the `-dr` or `--discresp` option: + +``` +▶ meg --discresp '": "v1"' /robots.txt +``` + ### Specifying The Method You can specify which HTTP method to use with the `-X` or `--method` option: diff --git a/args.go b/args.go index 90cca52..9551c07 100644 --- a/args.go +++ b/args.go @@ -44,9 +44,13 @@ type config struct { body string concurrency int delay int + discResp string headers headerArgs followLocation bool method string + regexIgnore string + regexKeep string + saveResp string saveStatus saveStatusArgs timeout int verbose bool @@ -76,6 +80,11 @@ func processArgs() config { flag.IntVar(&delay, "delay", 5000, "") flag.IntVar(&delay, "d", 5000, "") + // discResp params + discResp := "" + flag.StringVar(&discResp, "discresp", "", "") + flag.StringVar(&discResp, "dr", "", "") + // headers params var headers headerArgs flag.Var(&headers, "header", "") @@ -91,6 +100,21 @@ func processArgs() config { flag.StringVar(&method, "method", "GET", "") flag.StringVar(&method, "X", "GET", "") + // regexIgnore params + regexIgnore := "" + flag.StringVar(®exIgnore, "regexIgnore", "", "") + flag.StringVar(®exIgnore, "ri", "", "") + + // regexKeep params + regexKeep := "" + flag.StringVar(®exKeep, "regexKeep", "", "") + flag.StringVar(®exKeep, "rk", "", "") + + // saveResp params + saveResp := "" + flag.StringVar(&saveResp, "saveresp", "", "") + flag.StringVar(&saveResp, "sr", "", "") + // savestatus params var saveStatus saveStatusArgs flag.Var(&saveStatus, "savestatus", "") @@ -145,9 +169,13 @@ func processArgs() config { body: body, concurrency: concurrency, delay: delay, + discResp: discResp, headers: headers, followLocation: followLocation, method: method, + regexIgnore: regexIgnore, + regexKeep: regexKeep, + saveResp: saveResp, saveStatus: saveStatus, timeout: timeout, requester: requesterFn, @@ -167,16 +195,20 @@ func init() { h += " meg [path|pathsFile] [hostsFile] [outputDir]\n\n" h += "Options:\n" - h += " -b, --body Set the request body\n" - h += " -c, --concurrency Set the concurrency level (default: 20)\n" - h += " -d, --delay Milliseconds between requests to the same host (default: 5000)\n" - h += " -H, --header
Send a custom HTTP header\n" - h += " -L, --location Follow redirects / location header\n" - h += " -r, --rawhttp Use the rawhttp library for requests (experimental)\n" - h += " -s, --savestatus Save only responses with specific status code\n" - h += " -t, --timeout Set the HTTP timeout (default: 10000)\n" - h += " -v, --verbose Verbose mode\n" - h += " -X, --method HTTP method (default: GET)\n\n" + h += " -b, --body Set the request body\n" + h += " -c, --concurrency Set the concurrency level (default: 20)\n" + h += " -d, --delay Milliseconds between requests to the same host (default: 5000)\n" + h += " -dr, --discresp Discard responses containing specific string\n" + h += " -H, --header
Send a custom HTTP header\n" + h += " -L, --location Follow redirects / location header\n" + h += " -r, --rawhttp Use the rawhttp library for requests (experimental)\n" + h += " -ri, --regexignore Ignore results where the body matches a regex pattern\n\n" + h += " -rk, --regexkeep Keep results where the body matches a regex pattern\n\n" + h += " -sr, --saveresp Save only responses containing specific string\n" + h += " -s, --savestatus Save only responses with specific status code\n" + h += " -t, --timeout Set the HTTP timeout (default: 10000)\n" + h += " -v, --verbose Verbose mode\n" + h += " -X, --method HTTP method (default: GET)\n\n" h += "Defaults:\n" h += " pathsFile: ./paths\n" diff --git a/main.go b/main.go old mode 100644 new mode 100755 index f2a73a2..82aa197 --- a/main.go +++ b/main.go @@ -6,6 +6,8 @@ import ( "net/url" "os" "path/filepath" + "regexp" + "strings" "sync" "time" ) @@ -88,6 +90,29 @@ func main() { continue } + if len(c.saveResp) > 0 && !(strings.Contains(strings.Join(res.headers, ""), c.saveResp) || (strings.Contains(string(res.body), c.saveResp))) { + continue + } + + if len(c.discResp) > 0 && (strings.Contains(strings.Join(res.headers, ""), c.discResp) || (strings.Contains(string(res.body), c.discResp))) { + continue + } + if len(c.regexIgnore) > 0 { + re := regexp.MustCompile(c.regexIgnore) + matched := re.MatchString(res.String()) + if matched { + continue + } + } + + if len(c.regexKeep) > 0 { + re := regexp.MustCompile(c.regexKeep) + matched := re.MatchString(res.String()) + if !matched { + continue + } + } + if res.err != nil { fmt.Fprintf(os.Stderr, "request failed: %s\n", res.err) continue