Inspect robots policy

Fetches an origin's robots.txt and evaluates one path for a named user agent using longest-match allow/disallow precedence. Also returns crawl delay and declared sitemaps. Use this before automated retrieval; the result is policy guidance, not authorization. Limit: 8 s, 5 redirects, 768 KiB, 50 sitemap declarations.

When to use it

Fetches an origin's robots.txt and evaluates one path for a named user agent using longest-match allow/disallow precedence. Also returns crawl delay and declared sitemaps. Use this before automated retrieval; the result is policy guidance, not authorization.

Input schema

{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "type": "object",
  "properties": {
    "url": {
      "type": "string",
      "minLength": 8,
      "maxLength": 2048,
      "description": "Absolute public HTTP or HTTPS URL without embedded credentials."
    },
    "user_agent": {
      "default": "*",
      "type": "string",
      "minLength": 1,
      "maxLength": 200
    },
    "path": {
      "default": "/",
      "type": "string",
      "maxLength": 2048,
      "pattern": "^\\/.*"
    }
  },
  "required": [
    "url",
    "user_agent",
    "path"
  ],
  "additionalProperties": false
}

Output schema

{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "type": "object",
  "properties": {
    "ok": {
      "const": true
    },
    "data": {
      "type": "object",
      "properties": {
        "robots_url": {
          "type": "string"
        },
        "status": {
          "type": "integer"
        },
        "allowed": {
          "type": "boolean"
        },
        "matched_rule": {
          "type": [
            "object",
            "null"
          ]
        },
        "crawl_delay": {
          "type": [
            "number",
            "null"
          ]
        },
        "sitemaps": {
          "type": "array",
          "items": {
            "type": "string"
          }
        },
        "groups": {
          "type": "integer"
        }
      },
      "required": [
        "robots_url",
        "status",
        "allowed",
        "matched_rule",
        "crawl_delay",
        "sitemaps",
        "groups"
      ]
    },
    "meta": {
      "type": "object",
      "properties": {
        "request_id": {
          "type": "string"
        },
        "duration_ms": {
          "type": "integer"
        },
        "limits": {
          "type": "object"
        }
      },
      "required": [
        "request_id",
        "duration_ms"
      ]
    }
  },
  "required": [
    "ok",
    "data",
    "meta"
  ],
  "additionalProperties": false
}

REST example

curl -X POST https://plainport.exportitnow.workers.dev/api/v1/inspect-robots \
  -H "content-type: application/json" \
  -d '{"url":"https://example.com","path":"/docs","user_agent":"Plainport"}'

Example result

{
  "ok": true,
  "data": {
    "robots_url": "https://example.com/robots.txt",
    "status": 200,
    "allowed": true,
    "matched_rule": null
  },
  "meta": {
    "request_id": "018f…",
    "duration_ms": 12
  }
}

Limit