修改 Scrapy 目录为 telegram_groups_spider

.gitignore

@@ -1,4 +1,5 @@
 .DS_Store
 .idea
 .vscode
-venv
+venv
+__pycache__

spider/scrapy.cfg → telegram_groups_spider/scrapy.cfg

@@ -4,8 +4,8 @@
 # https://scrapyd.readthedocs.io/en/latest/deploy.html
 
 [settings]
-default = telegram_spider.settings
+default = telegram_groups_spider.settings
 
 [deploy]
 #url = http://localhost:6800/
-project = telegram_spider
+project = telegram_groups_spider

spider/telegram_spider/middlewares.py → telegram_groups_spider/telegram_groups_spider/middlewares.py

@@ -6,10 +6,9 @@
 from scrapy import signals
 
 # useful for handling different item types with a single interface
-from itemadapter import is_item, ItemAdapter
 
 
-class TelegramSpiderSpiderMiddleware:
+class TelegramGroupsSpiderSpiderMiddleware:
     # Not all methods need to be defined. If a method is not defined,
     # scrapy acts as if the spider middleware does not modify the
     # passed objects.
@@ -56,7 +55,7 @@ def spider_opened(self, spider):
         spider.logger.info("Spider opened: %s" % spider.name)
 
 
-class TelegramSpiderDownloaderMiddleware:
+class TelegramGroupsSpiderDownloaderMiddleware:
     # Not all methods need to be defined. If a method is not defined,
     # scrapy acts as if the downloader middleware does not modify the
     # passed objects.

spider/telegram_spider/settings.py → telegram_groups_spider/telegram_groups_spider/settings.py

@@ -12,7 +12,6 @@
 SPIDER_MODULES = ["telegram_spider.spiders"]
 NEWSPIDER_MODULE = "telegram_spider.spiders"
 
-
 # Crawl responsibly by identifying yourself (and your website) on the user-agent
 USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.5 Safari/605.1.15"
 
@@ -21,74 +20,74 @@
 
 LOG_LEVEL = "INFO"
 # Configure maximum concurrent requests performed by Scrapy (default: 16)
-#CONCURRENT_REQUESTS = 32
+# CONCURRENT_REQUESTS = 32
 
 # Configure a delay for requests for the same website (default: 0)
 # See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
 # See also autothrottle settings and docs
 # DOWNLOAD_DELAY = 5
 # The download delay setting will honor only one of:
-#CONCURRENT_REQUESTS_PER_DOMAIN = 16
-#CONCURRENT_REQUESTS_PER_IP = 16
+# CONCURRENT_REQUESTS_PER_DOMAIN = 16
+# CONCURRENT_REQUESTS_PER_IP = 16
 
 # Disable cookies (enabled by default)
-#COOKIES_ENABLED = False
+# COOKIES_ENABLED = False
 
 # Disable Telnet Console (enabled by default)
-#TELNETCONSOLE_ENABLED = False
+# TELNETCONSOLE_ENABLED = False
 
 # Override the default request headers:
-#DEFAULT_REQUEST_HEADERS = {
+# DEFAULT_REQUEST_HEADERS = {
 #    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
 #    "Accept-Language": "en",
-#}
+# }
 
 # Enable or disable spider middlewares
 # See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
-#SPIDER_MIDDLEWARES = {
+# SPIDER_MIDDLEWARES = {
 #    "telegram_spider.middlewares.TelegramSpiderSpiderMiddleware": 543,
-#}
+# }
 
 # Enable or disable downloader middlewares
 # See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
-#DOWNLOADER_MIDDLEWARES = {
+# DOWNLOADER_MIDDLEWARES = {
 #    "telegram_spider.middlewares.TelegramSpiderDownloaderMiddleware": 543,
-#}
+# }
 
 # Enable or disable extensions
 # See https://docs.scrapy.org/en/latest/topics/extensions.html
-#EXTENSIONS = {
+# EXTENSIONS = {
 #    "scrapy.extensions.telnet.TelnetConsole": None,
-#}
+# }
 
 # Configure item pipelines
 # See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
 ITEM_PIPELINES = {
-   "telegram_spider.pipelines.SavePipeline": 100,
-   # "telegram_spider.pipelines.ImagePipeline": 200,
-   # "telegram_spider.pipelines.PostgresqlPipeline": 300,
+    "telegram_spider.pipelines.SavePipeline": 100,
+    # "telegram_spider.pipelines.ImagePipeline": 200,
+    # "telegram_spider.pipelines.PostgresqlPipeline": 300,
 }
 
 # Enable and configure the AutoThrottle extension (disabled by default)
 # See https://docs.scrapy.org/en/latest/topics/autothrottle.html
-#AUTOTHROTTLE_ENABLED = True
+# AUTOTHROTTLE_ENABLED = True
 # The initial download delay
-#AUTOTHROTTLE_START_DELAY = 5
+# AUTOTHROTTLE_START_DELAY = 5
 # The maximum download delay to be set in case of high latencies
-#AUTOTHROTTLE_MAX_DELAY = 60
+# AUTOTHROTTLE_MAX_DELAY = 60
 # The average number of requests Scrapy should be sending in parallel to
 # each remote server
-#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
+# AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
 # Enable showing throttling stats for every response received:
-#AUTOTHROTTLE_DEBUG = False
+# AUTOTHROTTLE_DEBUG = False
 
 # Enable and configure HTTP caching (disabled by default)
 # See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
-#HTTPCACHE_ENABLED = True
-#HTTPCACHE_EXPIRATION_SECS = 0
-#HTTPCACHE_DIR = "httpcache"
-#HTTPCACHE_IGNORE_HTTP_CODES = []
-#HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"
+# HTTPCACHE_ENABLED = True
+# HTTPCACHE_EXPIRATION_SECS = 0
+# HTTPCACHE_DIR = "httpcache"
+# HTTPCACHE_IGNORE_HTTP_CODES = []
+# HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"
 
 # Set settings whose default value is deprecated to a future-proof value
 REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7"

spider/telegram_spider/spiders/telegram.py → telegram_groups_spider/telegram_groups_spider/spiders/telegram.py

@@ -1,7 +1,7 @@
 import scrapy
-from telegram_spider.items import TelegramSpiderItem
 from datetime import datetime
 from bs4 import BeautifulSoup
+from telegram_groups_spider.items import TelegramSpiderItem
 
 
 class BaseSpider(scrapy.Spider):
@@ -381,4 +381,4 @@ def parse(self, response, **kwargs):
         for tg_url in response.xpath(
                 '//article//a[starts-with(@href,"https://t.me")]/@href').getall():
             if tg_url:
-                yield scrapy.Request(tg_url, callback=self.parse_telegram_info)
+                yield scrapy.Request(tg_url, callback=self.parse_telegram_info)