From 4ec4758cfc934e96fe7f4ac0f0310200823f116e Mon Sep 17 00:00:00 2001 From: Jonas Lejon Date: Sat, 29 Aug 2026 17:47:03 +0200 Subject: [PATCH] feat: add scanmalware search agent ScanMalware searches an archive of sandboxed URL scans. The API is anonymous, so the agent works on a fresh install with no provider config; the query is SMQL, passed through unchanged the way the fofa and shodan agents pass theirs. The archive records what a browser saw, so results carry Host and Url rather than ip:port, which runner.go already handles by falling back to the host field, as it does for nerdydata. shodan-idb was the only keyless engine and was named as a string literal in two places. Those become an anonymousAgents set so a second keyless engine does not need a third and fourth literal. --- README.md | 5 +- runner/options.go | 14 +- sources/agent/scanmalware/response.go | 22 +++ sources/agent/scanmalware/scanmalware.go | 126 +++++++++++++ sources/agent/scanmalware/scanmalware_test.go | 169 ++++++++++++++++++ sources/session.go | 2 + uncover.go | 22 ++- 7 files changed, 352 insertions(+), 8 deletions(-) create mode 100644 sources/agent/scanmalware/response.go create mode 100644 sources/agent/scanmalware/scanmalware.go create mode 100644 sources/agent/scanmalware/scanmalware_test.go diff --git a/README.md b/README.md index f6a80d2e..3f6322e8 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,7 @@ - **[Driftnet](https://driftnet.io)** - **[DayDayMap](https://www.daydaymap.com)** - **[NerdyData](https://www.nerdydata.com/?utm_source=projectdiscovery/uncover)** + - **[ScanMalware](https://scanmalware.com)** - Multiple API key input support - Automatic API key randomization - **stdin** / **stdout** support for input @@ -79,12 +80,13 @@ Usage: Flags: INPUT: -q, -query string[] search query, supports: stdin,file,config input (example: -q 'example query', -q 'query.txt') - -e, -engine string[] search engine to query (shodan,shodan-idb,fofa,censys,quake,hunter,zoomeye,netlas,criminalip,publicwww,hunterhow,google,driftnet,daydaymap) (default shodan) + -e, -engine string[] search engine to query (shodan,shodan-idb,fofa,censys,quake,hunter,zoomeye,netlas,criminalip,publicwww,hunterhow,google,driftnet,daydaymap,scanmalware) (default shodan) -asq, -awesome-search-queries string[] use awesome search queries to discover exposed assets on the internet (example: -asq 'jira') SEARCH-ENGINE: -s, -shodan string[] search query for shodan (example: -shodan 'query.txt') -sd, -shodan-idb string[] search query for shodan-idb (example: -shodan-idb 'query.txt') + -sm, -scanmalware string[] search query for scanmalware (example: -scanmalware 'domain:*.example.com') -ff, -fofa string[] search query for fofa (example: -fofa 'query.txt') -cs, -censys string[] search query for censys (example: -censys 'query.txt') -qk, -quake string[] search query for quake (example: -quake 'query.txt') @@ -485,6 +487,7 @@ https://129.206.117.248 - `query` flag supports **all and only filters supported by search engine.** - results are limited to `100` as default and can be increased with `limit` flag. - `shodan-idb` API doesn't requires an API key and works out of the box. +- `scanmalware` API doesn't require an API key either. It searches an archive of sandboxed URL scans, so it returns URLs and hostnames rather than `ip:port`; `-f host` or `-f url` is the useful output here. - `shodan-idb` API is used as **default** engine when **IP/CIDR** is provided as input. ----- diff --git a/runner/options.go b/runner/options.go index 96c78a08..20f720c0 100644 --- a/runner/options.go +++ b/runner/options.go @@ -66,6 +66,7 @@ type Options struct { GreyNoise goflags.StringSlice Daydaymap goflags.StringSlice NerdyData goflags.StringSlice + ScanMalware goflags.StringSlice DisableUpdateCheck bool } @@ -77,7 +78,7 @@ func ParseOptions() *Options { flagSet.CreateGroup("input", "Input", flagSet.StringSliceVarP(&options.Query, "query", "q", nil, "search query, supports: stdin,file,config input (example: -q 'example query', -q 'query.txt')", goflags.FileStringSliceOptions), - flagSet.StringSliceVarP(&options.Engine, "engine", "e", nil, "search engine to query (shodan,shodan-idb,fofa,censys,quake,hunter,zoomeye,netlas,publicwww,criminalip,hunterhow,google,odin,binaryedge,onyphe,driftnet,greynoise,daydaymap,nerdydata) (default shodan)", goflags.FileNormalizedStringSliceOptions), + flagSet.StringSliceVarP(&options.Engine, "engine", "e", nil, "search engine to query (shodan,shodan-idb,fofa,censys,quake,hunter,zoomeye,netlas,publicwww,criminalip,hunterhow,google,odin,binaryedge,onyphe,driftnet,greynoise,daydaymap,nerdydata,scanmalware) (default shodan)", goflags.FileNormalizedStringSliceOptions), flagSet.StringSliceVarP(&options.AwesomeSearchQueries, "awesome-search-queries", "asq", nil, "use awesome search queries to discover exposed assets on the internet (example: -asq 'jira')", goflags.FileStringSliceOptions), ) @@ -101,6 +102,7 @@ func ParseOptions() *Options { flagSet.StringSliceVarP(&options.GreyNoise, "greynoise", "gn", nil, "search query for greynoise (example: -greynoise 'query.txt')", goflags.FileStringSliceOptions), flagSet.StringSliceVarP(&options.Daydaymap, "daydaymap", "ddm", nil, "search query for daydaymap (example: -daydaymap 'query.txt')", goflags.FileStringSliceOptions), flagSet.StringSliceVarP(&options.NerdyData, "nerdydata", "nd", nil, "search query for NerdyData (example: -nerdydata 'query.txt')", goflags.FileStringSliceOptions), + flagSet.StringSliceVarP(&options.ScanMalware, "scanmalware", "sm", nil, "search query for ScanMalware (example: -scanmalware 'domain:*.example.com')", goflags.FileStringSliceOptions), ) flagSet.CreateGroup("config", "Config", @@ -180,7 +182,8 @@ func ParseOptions() *Options { len(options.Driftnet), len(options.GreyNoise), len(options.Daydaymap), - len(options.NerdyData)) { + len(options.NerdyData), + len(options.ScanMalware)) { options.Engine = append(options.Engine, "shodan") } @@ -258,7 +261,8 @@ func (options *Options) validateOptions() error { len(options.Driftnet), len(options.GreyNoise), len(options.Daydaymap), - len(options.NerdyData)) { + len(options.NerdyData), + len(options.ScanMalware)) { return errors.New("no query provided") } @@ -299,7 +303,8 @@ func (options *Options) validateOptions() error { len(options.Driftnet), len(options.GreyNoise), len(options.Daydaymap), - len(options.NerdyData)) { + len(options.NerdyData), + len(options.ScanMalware)) { return errors.New("no engine specified") } @@ -346,6 +351,7 @@ func appendAllQueries(options *Options) { appendQuery(options, "greynoise", options.GreyNoise...) appendQuery(options, "daydaymap", options.Daydaymap...) appendQuery(options, "nerdydata", options.NerdyData...) + appendQuery(options, "scanmalware", options.ScanMalware...) } func (options *Options) useAwesomeSearchQueries(awesomeSearchQueries []string) error { diff --git a/sources/agent/scanmalware/response.go b/sources/agent/scanmalware/response.go new file mode 100644 index 00000000..868e1f4c --- /dev/null +++ b/sources/agent/scanmalware/response.go @@ -0,0 +1,22 @@ +package scanmalware + +// smqlResponse is one page of the ScanMalware archive search. +type smqlResponse struct { + Query string `json:"query"` + Results []smqlResult `json:"results"` +} + +// smqlResult is one scanned page. The archive records what a browser saw, so +// there is no IP or port here: `ip_count` and the ASN fields describe the +// addresses the scan resolved, not a single listening service. +type smqlResult struct { + ScanID string `json:"scan_id"` + URL string `json:"url"` + FinalURL string `json:"final_url"` + Title string `json:"title"` + Status string `json:"status"` + PrimaryASN string `json:"primary_asn"` + ASNNames []string `json:"asn_names"` + Countries []string `json:"countries"` + IPCount int `json:"ip_count"` +} diff --git a/sources/agent/scanmalware/scanmalware.go b/sources/agent/scanmalware/scanmalware.go new file mode 100644 index 00000000..dd4009f3 --- /dev/null +++ b/sources/agent/scanmalware/scanmalware.go @@ -0,0 +1,126 @@ +// Package scanmalware queries the ScanMalware archive of sandboxed URL scans. +// +// The query is SMQL, ScanMalware's own search syntax, passed through unchanged, +// the same way the fofa and shodan agents pass theirs. Examples: +// +// uncover -scanmalware 'domain:*.example.com' +// uncover -scanmalware 'asn:15169' +// +// The API is anonymous, so this agent runs on a fresh install with no +// provider-config.yaml, like shodan-idb. +package scanmalware + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + + "github.com/projectdiscovery/uncover/sources" +) + +var ( + // URL is a var rather than a const so tests can point it at httptest. + URL = "https://scanmalware.com/api/v1/search/smql?q=%s&limit=%d&page=%d" +) + +const ( + // pageSize is the maximum the endpoint accepts. It rejects larger values + // rather than clamping them. + pageSize = 100 + // maxPages bounds a broad query so one agent cannot stall a whole run. + maxPages = 100 + // userAgent identifies uncover to ScanMalware, so this traffic is + // attributable rather than looking like scraping. + userAgent = "scanmalware-uncover/1.0 (+https://github.com/projectdiscovery/uncover)" +) + +type Agent struct{} + +func (agent *Agent) Name() string { + return "scanmalware" +} + +func (agent *Agent) Query(ctx context.Context, session *sources.Session, query *sources.Query) (chan sources.Result, error) { + results := make(chan sources.Result) + + go func() { + defer close(results) + agent.query(ctx, session, query, results) + }() + + return results, nil +} + +func (agent *Agent) query(ctx context.Context, session *sources.Session, query *sources.Query, results chan sources.Result) { + sent := 0 + + for page := 1; page <= maxPages; page++ { + if ctx.Err() != nil { + return + } + + resp, err := agent.queryURL(ctx, session, query.Query, page) + if err != nil { + sources.SendResult(ctx, results, sources.Result{Source: agent.Name(), Error: err}) + return + } + + var body smqlResponse + err = json.NewDecoder(resp.Body).Decode(&body) + if closeErr := resp.Body.Close(); closeErr != nil && err == nil { + err = closeErr + } + if err != nil { + sources.SendResult(ctx, results, sources.Result{Source: agent.Name(), Error: err}) + return + } + + for _, row := range body.Results { + if sent >= query.Limit { + return + } + + result := sources.Result{Source: agent.Name(), Url: row.URL} + result.Host = hostOf(row.URL) + if result.Host == "" { + result.Host = row.URL + } + result.Raw, _ = json.Marshal(row) + + if !sources.SendResult(ctx, results, result) { + return + } + sent++ + } + + // A short page is the last page. + if len(body.Results) < pageSize { + return + } + } +} + +func (agent *Agent) queryURL(ctx context.Context, session *sources.Session, smql string, page int) (*http.Response, error) { + requestURL := fmt.Sprintf(URL, url.QueryEscape(smql), pageSize, page) + + request, err := sources.NewHTTPRequest(ctx, http.MethodGet, requestURL, nil) + if err != nil { + return nil, err + } + request.Header.Set("User-Agent", userAgent) + request.Header.Set("x-uncover", "1") + + return session.Do(request, agent.Name()) +} + +// hostOf returns the hostname of a URL, or an empty string when it cannot be +// parsed. The archive stores what was submitted, which is not always well formed. +func hostOf(rawURL string) string { + parsed, err := url.Parse(rawURL) + if err != nil { + return "" + } + return parsed.Hostname() +} diff --git a/sources/agent/scanmalware/scanmalware_test.go b/sources/agent/scanmalware/scanmalware_test.go new file mode 100644 index 00000000..19e1095b --- /dev/null +++ b/sources/agent/scanmalware/scanmalware_test.go @@ -0,0 +1,169 @@ +package scanmalware + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/projectdiscovery/uncover/sources" + "github.com/stretchr/testify/require" +) + +func newSession(t *testing.T) *sources.Session { + t.Helper() + session, err := sources.NewSession(&sources.Keys{}, 0, 5, 60, []string{"scanmalware"}, time.Second, "") + require.NoError(t, err) + return session +} + +func serve(t *testing.T, handler http.HandlerFunc) { + t.Helper() + ts := httptest.NewServer(handler) + t.Cleanup(ts.Close) + + originalURL := URL + URL = ts.URL + "?q=%s&limit=%d&page=%d" + t.Cleanup(func() { URL = originalURL }) +} + +func drain(t *testing.T, ch chan sources.Result) []sources.Result { + t.Helper() + var out []sources.Result + for r := range ch { + out = append(out, r) + } + return out +} + +// The archive returns URLs, not host:port. uncover falls back to the host field +// when IP is empty, so Host must be the parsed hostname rather than the raw URL. +func TestQueryReportsHostAndUrl(t *testing.T) { + serve(t, func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, `{"query":"domain:*.example.com","results":[ + {"scan_id":"abc","url":"https://login.example.com/path?a=1","title":"t"}]}`) + }) + + ch, err := (&Agent{}).Query(context.Background(), newSession(t), &sources.Query{Query: "domain:*.example.com", Limit: 100}) + require.NoError(t, err) + + results := drain(t, ch) + require.Len(t, results, 1) + require.Equal(t, "login.example.com", results[0].Host) + require.Equal(t, "https://login.example.com/path?a=1", results[0].Url) + require.Equal(t, "scanmalware", results[0].Source) + require.NotEmpty(t, results[0].Raw) +} + +// A short page is the last page. Without that check the agent would keep asking +// for pages that no longer exist until it hit maxPages. +func TestQueryStopsOnShortPage(t *testing.T) { + var pages int + serve(t, func(w http.ResponseWriter, r *http.Request) { + pages++ + fmt.Fprint(w, `{"results":[{"url":"https://a.example.com"}]}`) + }) + + ch, err := (&Agent{}).Query(context.Background(), newSession(t), &sources.Query{Query: "x", Limit: 100}) + require.NoError(t, err) + drain(t, ch) + + require.Equal(t, 1, pages, "a page shorter than pageSize means there is nothing after it") +} + +// query.Limit is the user's -limit. Exceeding it wastes the caller's quota and +// their terminal, so the agent must stop mid-page, not merely between pages. +func TestQueryHonorsLimit(t *testing.T) { + full := make([]string, 0, pageSize) + for i := 0; i < pageSize; i++ { + full = append(full, fmt.Sprintf(`{"url":"https://h%d.example.com"}`, i)) + } + body := `{"results":[` + strings.Join(full, ",") + `]}` + serve(t, func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, body) }) + + ch, err := (&Agent{}).Query(context.Background(), newSession(t), &sources.Query{Query: "x", Limit: 3}) + require.NoError(t, err) + + require.Len(t, drain(t, ch), 3) +} + +// Producer goroutines must abort their pending channel sends when the caller +// cancels ctx. A plain `results <- r` cannot: with no reader left it blocks +// forever and leaks the goroutine, and with a reader it keeps delivering the +// rest of the page after the caller has already given up. +// +// The observable difference is how much arrives after cancel: sources.SendResult +// stops on the first send, a plain send finishes the whole page. +func TestQueryStopsSendingOnceContextIsCancelled(t *testing.T) { + full := make([]string, 0, pageSize) + for i := 0; i < pageSize; i++ { + full = append(full, fmt.Sprintf(`{"url":"https://h%d.example.com"}`, i)) + } + body := `{"results":[` + strings.Join(full, ",") + `]}` + serve(t, func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, body) }) + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + ch, err := (&Agent{}).Query(ctx, newSession(t), &sources.Query{Query: "x", Limit: 100000}) + require.NoError(t, err) + + select { + case _, ok := <-ch: + require.True(t, ok, "expected at least one result before cancel") + case <-time.After(2 * time.Second): + t.Fatal("agent produced no results within 2s") + } + + cancel() + + after := 0 + done := make(chan struct{}) + go func() { + for range ch { + after++ + } + close(done) + }() + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("agent goroutine did not exit after ctx cancel") + } + require.Less(t, after, 5, "agent kept producing after cancel: %d more results", after) +} + +// A body that is not JSON must surface as an error result rather than a silent +// empty answer, which would read to the user as "nothing found". +func TestQueryReportsDecodeFailure(t *testing.T) { + serve(t, func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, `gateway error`) }) + + ch, err := (&Agent{}).Query(context.Background(), newSession(t), &sources.Query{Query: "x", Limit: 10}) + require.NoError(t, err) + + results := drain(t, ch) + require.Len(t, results, 1) + require.Error(t, results[0].Error) +} + +func TestSourceMetadata(t *testing.T) { + require.Equal(t, "scanmalware", (&Agent{}).Name()) + var _ sources.Agent = &Agent{} +} + +func TestHostOf(t *testing.T) { + for in, want := range map[string]string{ + "https://a.example.com/x": "a.example.com", + "http://b.example.com": "b.example.com", + "not a url": "", + "": "", + } { + require.Equal(t, want, hostOf(in), in) + } + _ = json.Marshal +} diff --git a/sources/session.go b/sources/session.go index d8f758ef..c90e31bc 100644 --- a/sources/session.go +++ b/sources/session.go @@ -34,6 +34,8 @@ var DefaultRateLimits = map[string]*ratelimit.Options{ "driftnet": {Key: "driftnet", MaxCount: 5, Duration: time.Second}, "greynoise": {Key: "greynoise", MaxCount: 1, Duration: time.Second}, "nerdydata": {Key: "nerdydata", MaxCount: 1, Duration: time.Second}, + // Anonymous API, documented at 600 req/min; 5/s leaves headroom. + "scanmalware": {Key: "scanmalware", MaxCount: 5, Duration: time.Second}, } // Session handles session agent sessions diff --git a/uncover.go b/uncover.go index 1472cb72..605bef30 100644 --- a/uncover.go +++ b/uncover.go @@ -23,6 +23,7 @@ import ( "github.com/projectdiscovery/uncover/sources/agent/nerdydata" "github.com/projectdiscovery/uncover/sources/agent/publicwww" "github.com/projectdiscovery/uncover/sources/agent/quake" + "github.com/projectdiscovery/uncover/sources/agent/scanmalware" "github.com/projectdiscovery/uncover/sources/agent/shodan" "github.com/projectdiscovery/uncover/sources/agent/shodanidb" "github.com/projectdiscovery/uncover/sources/agent/zoomeye" @@ -98,6 +99,8 @@ func New(opts *Options) (*Service, error) { s.Agents = append(s.Agents, &daydaymap.Agent{}) case "nerdydata": s.Agents = append(s.Agents, &nerdydata.Agent{}) + case "scanmalware": + s.Agents = append(s.Agents, &scanmalware.Agent{}) } } s.Provider = sources.NewProvider() @@ -137,7 +140,7 @@ func (s *Service) Execute(ctx context.Context) (<-chan sources.Result, error) { agentLabel: for _, agent := range s.Agents { keys := s.Provider.GetKeys() - if keys.Empty() && agent.Name() != "shodan-idb" { + if keys.Empty() && !isAnonymousAgent(agent.Name()) { gologger.Error().Msgf(agent.Name(), "agent given but keys not found") continue agentLabel } @@ -202,7 +205,7 @@ func (s *Service) ExecuteWithCallback(ctx context.Context, callback func(result // AllAgents returns all supported uncover Agents func (s *Service) AllAgents() []string { return []string{ - "shodan", "censys", "fofa", "shodan-idb", "quake", "hunter", "zoomeye", "netlas", "criminalip", "publicwww", "hunterhow", "google", "odin", "binaryedge", "onyphe", "driftnet", "greynoise", "daydaymap", "nerdydata", + "shodan", "censys", "fofa", "shodan-idb", "quake", "hunter", "zoomeye", "netlas", "criminalip", "publicwww", "hunterhow", "google", "odin", "binaryedge", "onyphe", "driftnet", "greynoise", "daydaymap", "nerdydata", "scanmalware", } } @@ -219,6 +222,19 @@ func (s *Service) nilCheck() error { return nil } +// anonymousAgents query APIs that need no key, so a run selecting only these +// must not be rejected for having no keys configured. +var anonymousAgents = []string{"shodan-idb", "scanmalware"} + +func isAnonymousAgent(name string) bool { + return stringsutil.EqualFoldAny(name, anonymousAgents...) +} + func (s *Service) hasAnyAnonymousProvider() bool { - return stringsutil.EqualFoldAny("shodan-idb", s.Options.Agents...) + for _, agent := range s.Options.Agents { + if isAnonymousAgent(agent) { + return true + } + } + return false }