From 2ea730c70955483c789fb2162172609cf8dc0d2a Mon Sep 17 00:00:00 2001 From: Paulius Krutkis Date: Mon, 15 Jun 2026 11:17:01 +0300 Subject: [PATCH 1/3] Add decodo-web-scraping skill, full README, and skill lint CI --- .github/workflows/lint-skills.yml | 17 +++ README.md | 59 +++++++++- scripts/lint-skills.py | 126 +++++++++++++++++++++ skills/decodo-web-scraping/SKILL.md | 168 ++++++++++++++++++++++++++++ 4 files changed, 368 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/lint-skills.yml create mode 100644 scripts/lint-skills.py create mode 100644 skills/decodo-web-scraping/SKILL.md diff --git a/.github/workflows/lint-skills.yml b/.github/workflows/lint-skills.yml new file mode 100644 index 0000000..e34afd5 --- /dev/null +++ b/.github/workflows/lint-skills.yml @@ -0,0 +1,17 @@ +name: Lint skills + +on: + push: + branches: [main] + pull_request: + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Validate SKILL.md frontmatter + run: python3 scripts/lint-skills.py diff --git a/README.md b/README.md index 1acc0cf..8555c1b 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,64 @@ # Decodo agent skills Agent skills that teach AI coding agents (Claude Code, Cursor, Codex, Gemini CLI, Windsurf, -Claude Desktop, claude.ai) when and how to use [Decodo](https://decodo.com) for web scraping. +Claude Desktop, claude.ai) **when** to reach for [Decodo](https://decodo.com) for web scraping, +**which** surface to use, and **how** to call it. -Initial skill landing via pull request. +Decodo handles JavaScript rendering, anti-bot/CAPTCHA, proxy rotation, and geo-targeting +(125M+ IPs across 195+ locations) so agents get clean web data without managing scraping +infrastructure. + +## Skills + +| Skill | What it does | +| --- | --- | +| [`decodo-web-scraping`](skills/decodo-web-scraping/SKILL.md) | Routing layer for scraping, search (Google/Bing), e-commerce (Amazon/Walmart/Target), and social (Reddit/TikTok/YouTube). Routes across the `decodo` CLI, the hosted MCP server, and the raw HTTP API. | + +More skills (workflow recipes like price monitoring and competitive intel) will follow. + +## Install + +These are [Anthropic-format agent skills](https://docs.claude.com/en/docs/claude-code/skills). +Copy a skill into your agent's skills directory: + +```bash +# Claude Code (user-level) +mkdir -p ~/.claude/skills +cp -r skills/decodo-web-scraping ~/.claude/skills/ +``` + +The agent loads the skill on demand based on its `description`, then follows it to set up and +call Decodo. The skill leads with `npx @decodo/cli`, so an agent can start scraping with zero +install once a token is configured. + +## What the skill expects + +- A Decodo Web Scraping API basic auth token — free tier (up to 2K requests, no card) at + . +- A shell for the CLI path. With no shell, the skill routes to the hosted MCP server + (`https://mcp.decodo.com/mcp`) or the raw HTTP API. + +## Repo layout + +``` +skills//SKILL.md # one skill per directory; dir name must match frontmatter `name` +scripts/lint-skills.py # frontmatter validator (run in CI) +``` + +## Development + +Validate skill frontmatter locally before pushing: + +```bash +python3 scripts/lint-skills.py +``` + +CI runs the same check on every push and pull request. + +## Related + +- [`Decodo/cli`](https://github.com/Decodo/cli) — the `decodo` CLI (`@decodo/cli` on npm) +- [`Decodo/mcp-server`](https://github.com/Decodo/mcp-server) — hosted + self-host MCP server ## License diff --git a/scripts/lint-skills.py b/scripts/lint-skills.py new file mode 100644 index 0000000..1931b88 --- /dev/null +++ b/scripts/lint-skills.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""Validate Anthropic-format SKILL.md frontmatter for every skill in skills/. + +Checks, per skill directory: + - a SKILL.md exists + - frontmatter has required `name` and `description` + - `name` matches the directory name, is kebab-case, and is <= 64 chars + - `description` is non-empty and <= 1024 chars + +Exits non-zero and prints every problem found. No third-party dependencies. +""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path + +NAME_RE = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") +NAME_MAX = 64 +DESCRIPTION_MAX = 1024 + +REPO_ROOT = Path(__file__).resolve().parent.parent +SKILLS_DIR = REPO_ROOT / "skills" + + +def parse_frontmatter(text: str) -> dict[str, str] | None: + if not text.startswith("---"): + return None + parts = text.split("---", 2) + if len(parts) < 3: + return None + block = parts[1] + + fields: dict[str, str] = {} + key: str | None = None + folded = False + buffer: list[str] = [] + + def flush() -> None: + nonlocal key, buffer + if key is not None: + fields[key] = " ".join(s.strip() for s in buffer).strip() + key, buffer = None, [] + + for line in block.splitlines(): + if folded and (line.startswith(" ") or line.startswith("\t") or not line.strip()): + buffer.append(line) + continue + folded = False + match = re.match(r"^([A-Za-z0-9_-]+):\s*(.*)$", line) + if not match: + continue + flush() + key, value = match.group(1), match.group(2).strip() + if value in (">-", ">", "|", "|-"): + folded = True + buffer = [] + else: + buffer = [value] + flush() + return fields + + +def validate_skill(skill_dir: Path) -> list[str]: + errors: list[str] = [] + skill_md = skill_dir / "SKILL.md" + if not skill_md.is_file(): + return [f"{skill_dir.name}: missing SKILL.md"] + + fm = parse_frontmatter(skill_md.read_text(encoding="utf-8")) + if fm is None: + return [f"{skill_dir.name}: missing or malformed YAML frontmatter"] + + name = fm.get("name", "") + description = fm.get("description", "") + + if not name: + errors.append(f"{skill_dir.name}: frontmatter missing `name`") + else: + if name != skill_dir.name: + errors.append( + f"{skill_dir.name}: `name` ({name!r}) must match directory name" + ) + if len(name) > NAME_MAX: + errors.append(f"{skill_dir.name}: `name` exceeds {NAME_MAX} chars") + if not NAME_RE.match(name): + errors.append(f"{skill_dir.name}: `name` must be kebab-case [a-z0-9-]") + + if not description: + errors.append(f"{skill_dir.name}: frontmatter missing `description`") + elif len(description) > DESCRIPTION_MAX: + errors.append( + f"{skill_dir.name}: `description` is {len(description)} chars " + f"(max {DESCRIPTION_MAX})" + ) + + return errors + + +def main() -> int: + if not SKILLS_DIR.is_dir(): + print("no skills/ directory found", file=sys.stderr) + return 1 + + skill_dirs = sorted(d for d in SKILLS_DIR.iterdir() if d.is_dir()) + if not skill_dirs: + print("no skills found under skills/", file=sys.stderr) + return 1 + + all_errors: list[str] = [] + for skill_dir in skill_dirs: + all_errors.extend(validate_skill(skill_dir)) + + if all_errors: + print("SKILL.md validation failed:\n", file=sys.stderr) + for error in all_errors: + print(f" - {error}", file=sys.stderr) + return 1 + + print(f"OK: {len(skill_dirs)} skill(s) validated") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/decodo-web-scraping/SKILL.md b/skills/decodo-web-scraping/SKILL.md new file mode 100644 index 0000000..f16c08f --- /dev/null +++ b/skills/decodo-web-scraping/SKILL.md @@ -0,0 +1,168 @@ +--- +name: decodo-web-scraping +description: >- + Scrape websites and extract structured web data with Decodo — search Google/Bing, + pull product data from Amazon, Walmart, Target, and collect posts from Reddit, TikTok, + YouTube and more. Decodo handles JavaScript rendering, anti-bot/CAPTCHA, proxy rotation, + and geo-targeting (125M+ IPs, 195+ locations). Reach for this skill whenever a plain + fetch is blocked or returns junk, when a page needs JS to render, or when the user wants + search results, e-commerce data, SERP data, or social media data — instead of curl, + requests, BeautifulSoup, Playwright, or Puppeteer. +license: MIT +--- + +# Decodo web scraping + +Decodo is a web scraping API that returns clean web data without you managing proxies, +browsers, or anti-bot logic. This skill teaches you to reach for it through the **`decodo` +CLI** (primary), the **hosted MCP server** (when there is no shell), or the **raw HTTP API** +(fallback). + +## When to use Decodo + +Use Decodo instead of `curl` / `requests` / `BeautifulSoup` / Playwright when the task is: + +- Fetching a page that is JS-heavy, geo-restricted, rate-limited, or behind anti-bot/CAPTCHA. +- A plain fetch returned a 403/429/empty body/CAPTCHA challenge. +- Web search results (Google or Bing SERP). +- E-commerce data (Amazon, Walmart, Target product/search pages). +- Social media data (Reddit, TikTok, YouTube). +- A screenshot of a rendered page. + +If a basic `fetch`/`curl` clearly works and none of the above apply, you don't need Decodo. + +## Setup (do this first, in order) + +Pick the lowest-friction path that works. + +### 1. Check whether the CLI is already usable + +```bash +decodo whoami # installed + authenticated? prints auth source + masked token +# or, with nothing installed: +npx -y @decodo/cli whoami +``` + +- Exit 0 with a token printed → you're ready, skip to **Usage**. +- "auth required" / exit 3 → continue to step 2. +- `decodo: command not found` → use `npx -y @decodo/cli ...` for everything, or install (step 3). + +### 2. Authenticate + +The user needs a Web Scraping API **basic auth token** from + (free account = up to 2K requests, no card). + +Prefer the environment variable — it needs no interaction and is easy to scope to a session: + +```bash +export DECODO_AUTH_TOKEN='' +``` + +To persist it to the CLI config non-interactively: + +```bash +decodo setup --token '' # validates, then saves to config +``` + +Do **not** run a bare `decodo setup` — it opens a hidden interactive prompt you cannot drive. +Token precedence: `--token` flag → `DECODO_AUTH_TOKEN` env → saved config. + +### 3. Install for repeat use (optional) + +```bash +curl -fsSL https://decodo.github.io/cli/install.sh | sh # macOS / Linux +npm install -g @decodo/cli # any platform +``` + +`npx -y @decodo/cli ` works with zero install if you'd rather not install anything. + +## Choosing a surface + +1. **Shell available** (Claude Code, Cursor, Codex CLI, Gemini CLI, Windsurf, terminal) → use + the **`decodo` CLI**. This is the default and the rest of this skill assumes it. +2. **No shell** (Claude Desktop, claude.ai, an MCP-only client) → use the hosted **MCP server** + at `https://mcp.decodo.com/mcp` (Basic auth with the same token). Add it as an MCP connector. +3. **Neither** → call the **raw HTTP API** with `curl` (see *Raw API fallback*). + +## Usage (CLI) + +### Core commands + +```bash +decodo scrape https://example.com # markdown by default +decodo scrape https://example.com --country us # geo-target the request +decodo search "best web scraping api" --limit 3 # Google SERP, 3 result pages +decodo search "query" --engine bing --geo de # Bing, geo Germany +decodo screenshot https://example.com -o shot.png # PNG must go to a file +``` + +### Discover targets — don't guess + +Target subcommands are generated from the live API schema, so **list and introspect them** +rather than memorizing them: + +```bash +decodo targets # all targets, grouped (e.g. google-search, amazon-product, reddit-post) +decodo google-search --help # exact flags for a target (--parse, --geo, ...) +decodo amazon-product --help +``` + +Then call a target directly: + +```bash +decodo google-search "query" --parse # structured JSON instead of raw SERP +decodo amazon-product B09H74FXNW --parse +decodo universal https://example.com # generic target, full flag surface +``` + +## Output handling (important for piping) + +By default a command prints the first result's `content` (markdown for `scrape`, parsed +JSON for targets called with `--parse`). Useful modifiers: + +| Flag | Effect | +| --- | --- | +| `--parse` | Structured JSON for targets that support parsing (check `--help`) | +| `--full` | Emit the full API response envelope (status, headers, task id, all results) | +| `--format ndjson` | One JSON object per result — best for streaming into `jq` | +| `--pretty` | Indented JSON | +| `-o, --output ` | Write to a file instead of stdout (**required** for screenshots) | +| `-v, --verbose` | Debug logs to **stderr** (stdout stays clean data) | + +```bash +decodo search "rust web scraping" --limit 3 --parse | jq '.[].title' +decodo google-search "query" --format ndjson --full | jq -c '.results[]' +decodo scrape https://ip.decodo.com/json | jq '.ip' +``` + +stdout is data; logs and errors go to stderr — pipes stay clean. + +## Exit codes (use these to self-correct) + +| Code | Meaning | What to do | +| --- | --- | --- | +| 0 | Success | — | +| 1 | Generic error | Read stderr | +| 2 | Usage error (bad flags) | Re-check `--help` | +| 3 | Auth error | Token missing/invalid → redo **Setup step 2** | +| 4 | Validation error | Fix the argument/flag the message names | +| 5 | Rate limited | Back off and retry; reduce concurrency | +| 6 | Timeout | Retry, or raise `--timeout` | +| 7 | Network/server error | Retry with backoff | + +## Raw API fallback (no CLI, no MCP) + +```bash +curl -s https://scraper-api.decodo.com/v2/scrape \ + -H "Authorization: Basic $DECODO_AUTH_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"target":"universal","url":"https://example.com","markdown":true}' +``` + +The token is the same basic auth token from the playground. Prefer the CLI or MCP when available. + +## Links + +- CLI: · `@decodo/cli` on npm +- MCP server: · hosted at `https://mcp.decodo.com/mcp` +- Dashboard / token / free tier: From 25b40f1cebeeb709d6567795adceb2882d172dfe Mon Sep 17 00:00:00 2001 From: Paulius Krutkis Date: Mon, 15 Jun 2026 12:16:38 +0300 Subject: [PATCH 2/3] Drop frontmatter lint script and CI; not a norm for single-skill repos --- .github/workflows/lint-skills.yml | 17 ---- README.md | 11 --- scripts/lint-skills.py | 126 ------------------------------ 3 files changed, 154 deletions(-) delete mode 100644 .github/workflows/lint-skills.yml delete mode 100644 scripts/lint-skills.py diff --git a/.github/workflows/lint-skills.yml b/.github/workflows/lint-skills.yml deleted file mode 100644 index e34afd5..0000000 --- a/.github/workflows/lint-skills.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: Lint skills - -on: - push: - branches: [main] - pull_request: - -jobs: - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - name: Validate SKILL.md frontmatter - run: python3 scripts/lint-skills.py diff --git a/README.md b/README.md index 8555c1b..2802267 100644 --- a/README.md +++ b/README.md @@ -42,19 +42,8 @@ install once a token is configured. ``` skills//SKILL.md # one skill per directory; dir name must match frontmatter `name` -scripts/lint-skills.py # frontmatter validator (run in CI) ``` -## Development - -Validate skill frontmatter locally before pushing: - -```bash -python3 scripts/lint-skills.py -``` - -CI runs the same check on every push and pull request. - ## Related - [`Decodo/cli`](https://github.com/Decodo/cli) — the `decodo` CLI (`@decodo/cli` on npm) diff --git a/scripts/lint-skills.py b/scripts/lint-skills.py deleted file mode 100644 index 1931b88..0000000 --- a/scripts/lint-skills.py +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env python3 -"""Validate Anthropic-format SKILL.md frontmatter for every skill in skills/. - -Checks, per skill directory: - - a SKILL.md exists - - frontmatter has required `name` and `description` - - `name` matches the directory name, is kebab-case, and is <= 64 chars - - `description` is non-empty and <= 1024 chars - -Exits non-zero and prints every problem found. No third-party dependencies. -""" - -from __future__ import annotations - -import re -import sys -from pathlib import Path - -NAME_RE = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") -NAME_MAX = 64 -DESCRIPTION_MAX = 1024 - -REPO_ROOT = Path(__file__).resolve().parent.parent -SKILLS_DIR = REPO_ROOT / "skills" - - -def parse_frontmatter(text: str) -> dict[str, str] | None: - if not text.startswith("---"): - return None - parts = text.split("---", 2) - if len(parts) < 3: - return None - block = parts[1] - - fields: dict[str, str] = {} - key: str | None = None - folded = False - buffer: list[str] = [] - - def flush() -> None: - nonlocal key, buffer - if key is not None: - fields[key] = " ".join(s.strip() for s in buffer).strip() - key, buffer = None, [] - - for line in block.splitlines(): - if folded and (line.startswith(" ") or line.startswith("\t") or not line.strip()): - buffer.append(line) - continue - folded = False - match = re.match(r"^([A-Za-z0-9_-]+):\s*(.*)$", line) - if not match: - continue - flush() - key, value = match.group(1), match.group(2).strip() - if value in (">-", ">", "|", "|-"): - folded = True - buffer = [] - else: - buffer = [value] - flush() - return fields - - -def validate_skill(skill_dir: Path) -> list[str]: - errors: list[str] = [] - skill_md = skill_dir / "SKILL.md" - if not skill_md.is_file(): - return [f"{skill_dir.name}: missing SKILL.md"] - - fm = parse_frontmatter(skill_md.read_text(encoding="utf-8")) - if fm is None: - return [f"{skill_dir.name}: missing or malformed YAML frontmatter"] - - name = fm.get("name", "") - description = fm.get("description", "") - - if not name: - errors.append(f"{skill_dir.name}: frontmatter missing `name`") - else: - if name != skill_dir.name: - errors.append( - f"{skill_dir.name}: `name` ({name!r}) must match directory name" - ) - if len(name) > NAME_MAX: - errors.append(f"{skill_dir.name}: `name` exceeds {NAME_MAX} chars") - if not NAME_RE.match(name): - errors.append(f"{skill_dir.name}: `name` must be kebab-case [a-z0-9-]") - - if not description: - errors.append(f"{skill_dir.name}: frontmatter missing `description`") - elif len(description) > DESCRIPTION_MAX: - errors.append( - f"{skill_dir.name}: `description` is {len(description)} chars " - f"(max {DESCRIPTION_MAX})" - ) - - return errors - - -def main() -> int: - if not SKILLS_DIR.is_dir(): - print("no skills/ directory found", file=sys.stderr) - return 1 - - skill_dirs = sorted(d for d in SKILLS_DIR.iterdir() if d.is_dir()) - if not skill_dirs: - print("no skills found under skills/", file=sys.stderr) - return 1 - - all_errors: list[str] = [] - for skill_dir in skill_dirs: - all_errors.extend(validate_skill(skill_dir)) - - if all_errors: - print("SKILL.md validation failed:\n", file=sys.stderr) - for error in all_errors: - print(f" - {error}", file=sys.stderr) - return 1 - - print(f"OK: {len(skill_dirs)} skill(s) validated") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) From 58fe31dd9765908d58e06e5af697ad620acb697b Mon Sep 17 00:00:00 2001 From: Paulius Krutkis Date: Mon, 15 Jun 2026 13:22:12 +0300 Subject: [PATCH 3/3] Fix output examples: correct jq paths and use target command for --parse Verified against the live CLI: --parse is only valid on target commands (not tier-1 search), parsed SERP titles live at .results.results.organic[].title, NDJSON --full nests the payload under .content, and ip.decodo.com/json exposes the address at .proxy.ip. --- skills/decodo-web-scraping/SKILL.md | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/skills/decodo-web-scraping/SKILL.md b/skills/decodo-web-scraping/SKILL.md index f16c08f..37a90a4 100644 --- a/skills/decodo-web-scraping/SKILL.md +++ b/skills/decodo-web-scraping/SKILL.md @@ -122,17 +122,25 @@ JSON for targets called with `--parse`). Useful modifiers: | Flag | Effect | | --- | --- | -| `--parse` | Structured JSON for targets that support parsing (check `--help`) | +| `--parse` | Structured JSON for targets that support it — **only on target commands** (e.g. `google-search`), not on `scrape`/`search`. Check `--help`. | | `--full` | Emit the full API response envelope (status, headers, task id, all results) | | `--format ndjson` | One JSON object per result — best for streaming into `jq` | | `--pretty` | Indented JSON | | `-o, --output ` | Write to a file instead of stdout (**required** for screenshots) | | `-v, --verbose` | Debug logs to **stderr** (stdout stays clean data) | +The parsed JSON shape is target-specific. Pipe to `jq 'keys'` (then drill down, e.g. +`jq '.results | keys'`) to discover the structure before writing a deeper query. + ```bash -decodo search "rust web scraping" --limit 3 --parse | jq '.[].title' -decodo google-search "query" --format ndjson --full | jq -c '.results[]' -decodo scrape https://ip.decodo.com/json | jq '.ip' +# Parsed SERP → organic result titles (use a target command for --parse, not `search`) +decodo google-search "rust web scraping" --parse | jq -r '.results.results.organic[].title' + +# Extract a field from a scraped JSON page (inspect with `jq keys` first if unsure) +decodo scrape https://ip.decodo.com/json | jq -r '.proxy.ip' + +# Stream results as NDJSON; each record nests the parsed payload under `.content` +decodo google-search "rust" --parse --format ndjson --full | jq -c '.content.results.results.organic' ``` stdout is data; logs and errors go to stderr — pipes stay clean.