Domain-Scoped Site Crawler for Markdown Datasets

{
  "description": "Enter one or more start URLs, set your crawl and content options, and run the Actor to get clean Markdown, text, or HTML per page — with metadata, links, and images. The Actor fetches each page the fastest way that works automatically (plain HTTP, browser TLS fingerprint, or a headless browser for JavaScript pages).",
  "properties": {
    "chunkMarkdown": {
      "default": false,
      "description": "Emit a chunks array per page — heading-aware Markdown pieces ready for embeddings. Code blocks and tables are never split, each piece carries its heading path and character count, and consecutive pieces overlap so context is kept at the boundaries.",
      "prefill": false,
      "title": "Split Content into Chunks",
      "type": "boolean"
    },
    "chunkMaxChars": {
      "default": 2000,
      "description": "Target size of each chunk when chunkMarkdown is on. Code blocks and tables stay whole even when larger than this.",
      "maximum": 10000,
      "minimum": 200,
      "prefill": 2000,
      "title": "Chunk Size (characters)",
      "type": "integer"
    },
    "chunkOverlapChars": {
      "default": 200,
      "description": "Approximate overlap between consecutive chunks when chunkMarkdown is on, so context is not lost at chunk boundaries.",
      "maximum": 1000,
      "minimum": 0,
      "prefill": 200,
      "title": "Chunk Overlap (characters)",
      "type": "integer"
    },
    "clickElementsCssSelector": {
      "default": "",
      "description": "Click elements matching this CSS selector (e.g. [aria-expanded=\"false\"]) before extraction to expand collapsed content.",
      "editor": "textfield",
      "prefill": "",
      "sectionCaption": "Rendered pages",
      "sectionDescription": "Options for content that only appears after JavaScript runs.",
      "title": "Click Elements (CSS selector)",
      "type": "string"
    },
    "crawlOnlyPagesChangedSince": {
      "default": "",
      "description": "Optional date (YYYY-MM-DD or full ISO date-time). Crawls only pages that may have changed since then and outputs a delta. Leave empty to crawl everything.",
      "editor": "textfield",
      "prefill": "",
      "title": "Crawl Only Pages Changed Since",
      "type": "string"
    },
    "customHttpHeaders": {
      "default": [],
      "description": "Extra request headers for all engines and discovery requests, as a list of {\"key\", \"value\"} pairs (e.g. an Authorization token or a specific User-Agent for sites that ask for one). Transport/framing and cookie headers (Host, Cookie, Content-Length, Connection, ...) are rejected so the actor stays cookieless.",
      "editor": "json",
      "items": {
        "properties": {
          "key": {
            "description": "HTTP header name, e.g. Authorization or Accept-Language.",
            "title": "Header name",
            "type": "string"
          },
          "value": {
            "description": "Header value sent with every request.",
            "title": "Header value",
            "type": "string"
          }
        },
        "required": [
          "key",
          "value"
        ],
        "type": "object"
      },
      "prefill": [],
      "sectionCaption": "Advanced",
      "sectionDescription": "Request headers and run diagnostics.",
      "title": "Custom HTTP Headers",
      "type": "array"
    },
    "excludeUrlGlobs": {
      "default": [],
      "description": "Skip discovered URLs matching any of these glob patterns (e.g. *login*, *.pdf).",
      "editor": "globs",
      "prefill": [],
      "title": "Exclude URL Patterns (globs)",
      "type": "array"
    },
    "excludeUrlRegexps": {
      "default": [],
      "description": "Skip discovered URLs matching any of these regular expressions.",
      "editor": "stringList",
      "prefill": [],
      "title": "Exclude URL Patterns (regex)",
      "type": "array"
    },
    "htmlTransformer": {
      "default": "auto",
      "description": "How the main content is isolated from navigation, footers, ads, and cookie banners.",
      "editor": "select",
      "enum": [
        "auto",
        "trafilatura",
        "readability",
        "basic",
        "none"
      ],
      "enumTitles": [
        "Auto (recommended) — best content, boilerplate removed",
        "Trafilatura (article/text extraction)",
        "Readability (Mozilla article extractor)",
        "Basic cleanup (remove nav/header/footer only)",
        "No cleaning (keep the full page)"
      ],
      "prefill": "auto",
      "title": "HTML Transformer",
      "type": "string"
    },
    "ignoreCanonicalUrl": {
      "default": false,
      "description": "When disabled, pages whose canonical URL points to a page that was already crawled are skipped as duplicates.",
      "prefill": false,
      "title": "Ignore canonical URL",
      "type": "boolean"
    },
    "includeImages": {
      "default": false,
      "description": "Add image references to the markdown field. Image URLs are always collected in the imageUrls field.",
      "prefill": false,
      "title": "Include Images in Markdown",
      "type": "boolean"
    },
    "includeUrlGlobs": {
      "default": [],
      "description": "Only crawl discovered URLs matching these glob patterns (e.g. https://example.com/docs/*). Leave empty to allow everything on the domain.",
      "editor": "globs",
      "prefill": [],
      "title": "Include URL Patterns (globs)",
      "type": "array"
    },
    "includeUrlRegexps": {
      "default": [],
      "description": "Only crawl discovered URLs matching at least one of these regular expressions. Combined with glob patterns (both must pass when both are set).",
      "editor": "stringList",
      "prefill": [],
      "title": "Include URL Patterns (regex)",
      "type": "array"
    },
    "keepElementsCssSelector": {
      "default": "",
      "description": "Keep only the elements matching this CSS selector (e.g. article, main .content) before extraction. Leave empty to use the whole page.",
      "editor": "textfield",
      "prefill": "",
      "title": "Keep HTML Elements (CSS selector)",
      "type": "string"
    },
    "keepUrlFragments": {
      "default": false,
      "description": "Treat URLs that differ only in #fragment as different pages. Enable for single-page sites that use fragments as routes.",
      "prefill": false,
      "title": "Keep URL Fragments",
      "type": "boolean"
    },
    "maxCrawlDepth": {
      "default": 10,
      "description": "How many links deep to follow from the start URLs. 0 means only the start URLs are crawled.",
      "maximum": 100,
      "minimum": 0,
      "prefill": 1,
      "sectionCaption": "Crawling",
      "sectionDescription": "Where the crawl starts and how far it goes.",
      "title": "Max Crawl Depth",
      "type": "integer"
    },
    "maxCrawlPages": {
      "default": 20,
      "description": "Maximum number of pages (and files) to store. The crawl stops when this limit is reached.",
      "maximum": 100000,
      "minimum": 1,
      "prefill": 3,
      "title": "Max Pages",
      "type": "integer"
    },
    "outputFormat": {
      "default": "markdown",
      "description": "How page content is returned. Markdown keeps headings, lists, tables, and links — best for LLMs and RAG.",
      "editor": "select",
      "enum": [
        "markdown",
        "text",
        "html"
      ],
      "enumTitles": [
        "Markdown",
        "Plain text",
        "Cleaned HTML"
      ],
      "prefill": "markdown",
      "sectionCaption": "Content extraction",
      "sectionDescription": "How page content is cleaned and returned.",
      "title": "Output Format",
      "type": "string"
    },
    "parsePdfText": {
      "default": true,
      "description": "When a crawled URL serves a PDF, extract its text into the text field as a file record instead of skipping it.",
      "prefill": true,
      "title": "Extract PDF Text",
      "type": "boolean"
    },
    "removeElementsCssSelector": {
      "default": "",
      "description": "Remove elements matching this CSS selector (e.g. .sidebar, .ad, nav) before extraction.",
      "editor": "textfield",
      "prefill": "",
      "title": "Remove HTML Elements (CSS selector)",
      "type": "string"
    },
    "respectRobotsTxtFile": {
      "default": true,
      "description": "Skip discovered pages that the site's robots.txt disallows. Start URLs are always crawled.",
      "prefill": true,
      "title": "Respect robots.txt",
      "type": "boolean"
    },
    "startUrls": {
      "description": "One or more URLs to start crawling from. The crawler follows links found on these pages and stays on the domain of the first start URL.",
      "editor": "requestListSources",
      "prefill": [
        {
          "url": "https://docs.apify.com"
        }
      ],
      "title": "Start URLs",
      "type": "array"
    },
    "storeSkippedUrls": {
      "default": false,
      "description": "Save a SKIPPED_URLS record to the key-value store listing skipped URLs grouped by reason (robots, blocked, non-HTML, errors, duplicates).",
      "prefill": false,
      "title": "Store Skipped URLs",
      "type": "boolean"
    },
    "takeScreenshots": {
      "default": false,
      "description": "Render every stored page in the browser and capture a full-page JPEG (quality 80) of it, putting its public URL in the screenshot field. Screenshots are held in memory until pushed and add roughly 0.3-1 MB and a fraction of a second per page; screenshot capture always runs after the page is rendered.",
      "prefill": false,
      "title": "Take Page Screenshots",
      "type": "boolean"
    },
    "useLlmsTxt": {
      "default": false,
      "description": "Look for /llms.txt at the start URL domains and enqueue the files listed there (common on documentation sites).",
      "prefill": false,
      "title": "Use llms.txt",
      "type": "boolean"
    },
    "useSitemaps": {
      "default": false,
      "description": "Look for sitemaps (from robots.txt and /sitemap.xml) and enqueue the URLs they list. Finds pages that are not linked from the start URLs.",
      "prefill": false,
      "sectionCaption": "Discovery & filters",
      "sectionDescription": "How the crawler finds pages and which URLs it keeps.",
      "title": "Use Sitemaps",
      "type": "boolean"
    }
  },
  "required": [
    "startUrls"
  ],
  "schemaVersion": 1,
  "title": "Domain-Scoped Site Crawler Input",
  "type": "object"
}

{
  "actorOutputSchemaVersion": 1,
  "description": "One record per crawled page: Markdown/text/HTML content, title, description, author, dates, canonical URL, links, images, OpenGraph, JSON-LD, word count, approximate token count, optional RAG chunks, optional screenshot URL, and content hash.",
  "properties": {
    "pages": {
      "description": "Dataset containing all crawled pages and extracted files.",
      "template": "{{links.apiDefaultDatasetUrl}}/items",
      "title": "Crawled pages",
      "type": "string"
    }
  },
  "title": "Domain-Scoped Site Crawler for Markdown Datasets Output"
}

{
  "actorSpecification": 1,
  "views": {
    "overview": {
      "display": {
        "component": "table",
        "properties": {
          "depth": {
            "format": "number",
            "label": "Depth"
          },
          "httpStatusCode": {
            "format": "number",
            "label": "Status"
          },
          "title": {
            "format": "text",
            "label": "Title"
          },
          "url": {
            "format": "text",
            "label": "URL"
          },
          "wordCount": {
            "format": "number",
            "label": "Words"
          }
        }
      },
      "title": "Crawled Pages",
      "transformation": {
        "fields": [
          "url",
          "title",
          "depth",
          "wordCount",
          "httpStatusCode"
        ]
      }
    }
  }
}