apache_conf config.json

Posted 2021-05-23

{
  "spiders": [
    {
      "name": "china",
      "custom_settings": null,
      "code": {},
      "extractors": [
        {
          "callback": "process_item",
          "item": "NewsItem",
          "attrs": {
            "title": [
              {
                "method": "css",
                "arg": "#chan_newsTitle::text",
                "processor": ""
              }
            ],
            "content": [
              {
                "method": "css",
                "arg": "#chan_newsDetail p::text",
                "processor": "Join()"
              }
            ],
            "datetime": [
              {
                "method": "css",
                "arg": "#chan_newsInfo::text",
                "re": "\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}",
                "processor": ""
              }
            ],
            "url": [
              {
                "method": "attr",
                "arg": "url"
              }
            ]
          }
        }
      ],
      "rules": [
        {
          "restrict_css": [
            ".con_item h2"
          ],
          "callback": "process_item"
        },
        {
          "restrict_xpaths": [
            "//div[@id=\"pageStyle\"]/a[contains(., \"下一页\")]"
          ]
        }
      ],
      "storage": {
        "mysql": {
          "enable": false
        }
      },
      "start_urls": {
        "mode": "list",
        "list": [
          "http://tech.china.com/internet/"
        ],
        "code": null,
        "file": null
      },
      "attrs": [],
      "allowed_domains": [
        "tech.china.com"
      ]
    }
  ],
  "items": [
    {
      "name": "NewsItem",
      "attrs": {
        "title": {
          "in_processor": "",
          "out_processor": "TakeFirst()"
        },
        "content": {
          "in_processor": "",
          "out_processor": "TakeFirst()"
        },
        "datetime": {
          "in_processor": "",
          "out_processor": "TakeFirst()"
        },
        "url": {
          "out_processor": "TakeFirst()"
        }
      }
    }
  ]
}