{ "schema_version": 1, "generated_at": "2026-09-25T16:13:49", "title": "AI crawler hits on our sites", "definition": "Daily counts of requests from AI answer-engine crawlers and traditional search crawlers to infogap.store and shengminglian.com.cn, broken down by user agent and by URL. One row = one day. One entry under a user agent = that agent that day.", "method": "Parsed from our own nginx combined access logs on the two servers, including rotated .gz files, by /opt/geo/ai_crawler_stats.py. Agents are matched by user-agent substring. Status codes are the codes our servers actually returned: a 404 means the crawler asked for something we do not serve.", "why": "This is the thermometer for our own GEO work: if an AI engine never fetches us, no amount of new pages will get us cited. 404 counts are the honest half of the picture - they show what crawlers came looking for and did not find.", "license": "CC BY 4.0 - attribution: InfoGap (infogap.store), link to this file.", "caution": "Counts come from access logs only. A user agent string can be spoofed, so a \"Googlebot\" line is not proof of Google.", "sites": { "infogap.store": { "generated_at": "2026-09-25T16:10:54.179723+08:00", "label": "SG", "hostname": "VM-0-11-ubuntu", "log_dir": "/var/log/nginx", "log_bases": [ "infogap.access.log", "access.log" ], "days_filter": 30, "window": { "start": "2026-09-12T11:28:32+08:00", "end": "2026-09-25T16:10:50+08:00", "reference_day": "2026-09-25", "cutoff_day": "2026-08-27" }, "total_lines": 103222, "parsed_lines": 103222, "ai_hits": 1722, "search_hits": 2868, "files_scanned": [ { "file": "/var/log/nginx/access.log", "bytes": 106678, "lines": 507, "parsed_lines": 507, "crawler_hits": 0, "window_start": "2026-09-25T00:00:22+08:00", "window_end": "2026-09-25T16:10:27+08:00" }, { "file": "/var/log/nginx/access.log.1", "bytes": 338338, "lines": 1676, "parsed_lines": 1676, "crawler_hits": 304, "window_start": "2026-09-24T00:01:23+08:00", "window_end": "2026-09-24T23:55:19+08:00" }, { "file": "/var/log/nginx/access.log.10.gz", "bytes": 15724, "lines": 957, "parsed_lines": 957, "crawler_hits": 0, "window_start": "2026-09-15T00:00:08+08:00", "window_end": "2026-09-15T23:47:48+08:00" }, { "file": "/var/log/nginx/access.log.11.gz", "bytes": 26597, "lines": 1890, "parsed_lines": 1890, "crawler_hits": 2, "window_start": "2026-09-14T00:00:25+08:00", "window_end": "2026-09-14T23:59:14+08:00" }, { "file": "/var/log/nginx/access.log.12.gz", "bytes": 33952, "lines": 1859, "parsed_lines": 1859, "crawler_hits": 340, "window_start": "2026-09-12T11:28:32+08:00", "window_end": "2026-09-13T23:59:52+08:00" }, { "file": "/var/log/nginx/access.log.2.gz", "bytes": 19788, "lines": 1543, "parsed_lines": 1543, "crawler_hits": 300, "window_start": "2026-09-23T00:03:52+08:00", "window_end": "2026-09-23T23:56:00+08:00" }, { "file": "/var/log/nginx/access.log.3.gz", "bytes": 29990, "lines": 2017, "parsed_lines": 2017, "crawler_hits": 10, "window_start": "2026-09-22T00:05:41+08:00", "window_end": "2026-09-22T23:57:37+08:00" }, { "file": "/var/log/nginx/access.log.4.gz", "bytes": 19110, "lines": 1414, "parsed_lines": 1414, "crawler_hits": 0, "window_start": "2026-09-21T00:01:41+08:00", "window_end": "2026-09-21T23:56:19+08:00" }, { "file": "/var/log/nginx/access.log.5.gz", "bytes": 18293, "lines": 1222, "parsed_lines": 1222, "crawler_hits": 0, "window_start": "2026-09-20T00:05:35+08:00", "window_end": "2026-09-20T23:58:07+08:00" }, { "file": "/var/log/nginx/access.log.6.gz", "bytes": 27904, "lines": 2359, "parsed_lines": 2359, "crawler_hits": 622, "window_start": "2026-09-19T00:01:39+08:00", "window_end": "2026-09-19T23:50:35+08:00" }, { "file": "/var/log/nginx/access.log.7.gz", "bytes": 17484, "lines": 1213, "parsed_lines": 1213, "crawler_hits": 31, "window_start": "2026-09-18T00:07:57+08:00", "window_end": "2026-09-18T23:43:01+08:00" }, { "file": "/var/log/nginx/access.log.8.gz", "bytes": 15863, "lines": 978, "parsed_lines": 978, "crawler_hits": 0, "window_start": "2026-09-17T00:04:06+08:00", "window_end": "2026-09-17T23:54:02+08:00" }, { "file": "/var/log/nginx/access.log.9.gz", "bytes": 14564, "lines": 884, "parsed_lines": 884, "crawler_hits": 0, "window_start": "2026-09-16T00:02:36+08:00", "window_end": "2026-09-16T23:48:55+08:00" }, { "file": "/var/log/nginx/infogap.access.log", "bytes": 1008826, "lines": 7568, "parsed_lines": 7568, "crawler_hits": 486, "window_start": "2026-09-25T00:00:13+08:00", "window_end": "2026-09-25T16:10:50+08:00" }, { "file": "/var/log/nginx/infogap.access.log.1", "bytes": 920052, "lines": 7529, "parsed_lines": 7529, "crawler_hits": 336, "window_start": "2026-09-24T00:00:08+08:00", "window_end": "2026-09-24T23:56:30+08:00" }, { "file": "/var/log/nginx/infogap.access.log.10.gz", "bytes": 17517, "lines": 1509, "parsed_lines": 1509, "crawler_hits": 1, "window_start": "2026-09-15T00:03:42+08:00", "window_end": "2026-09-15T23:29:55+08:00" }, { "file": "/var/log/nginx/infogap.access.log.11.gz", "bytes": 37901, "lines": 3261, "parsed_lines": 3261, "crawler_hits": 3, "window_start": "2026-09-14T00:00:25+08:00", "window_end": "2026-09-14T23:53:56+08:00" }, { "file": "/var/log/nginx/infogap.access.log.12.gz", "bytes": 43834, "lines": 2639, "parsed_lines": 2639, "crawler_hits": 704, "window_start": "2026-09-12T11:31:58+08:00", "window_end": "2026-09-13T23:53:19+08:00" }, { "file": "/var/log/nginx/infogap.access.log.2.gz", "bytes": 74015, "lines": 9134, "parsed_lines": 9134, "crawler_hits": 409, "window_start": "2026-09-23T00:00:17+08:00", "window_end": "2026-09-23T23:59:58+08:00" }, { "file": "/var/log/nginx/infogap.access.log.3.gz", "bytes": 63325, "lines": 7902, "parsed_lines": 7902, "crawler_hits": 221, "window_start": "2026-09-22T00:00:03+08:00", "window_end": "2026-09-22T23:59:06+08:00" }, { "file": "/var/log/nginx/infogap.access.log.4.gz", "bytes": 76112, "lines": 10514, "parsed_lines": 10514, "crawler_hits": 30, "window_start": "2026-09-21T00:00:16+08:00", "window_end": "2026-09-21T23:56:37+08:00" }, { "file": "/var/log/nginx/infogap.access.log.5.gz", "bytes": 132866, "lines": 18307, "parsed_lines": 18307, "crawler_hits": 12, "window_start": "2026-09-20T00:00:08+08:00", "window_end": "2026-09-20T23:56:30+08:00" }, { "file": "/var/log/nginx/infogap.access.log.6.gz", "bytes": 63686, "lines": 9099, "parsed_lines": 9099, "crawler_hits": 603, "window_start": "2026-09-19T00:00:20+08:00", "window_end": "2026-09-19T23:56:24+08:00" }, { "file": "/var/log/nginx/infogap.access.log.7.gz", "bytes": 21759, "lines": 1782, "parsed_lines": 1782, "crawler_hits": 79, "window_start": "2026-09-18T00:07:15+08:00", "window_end": "2026-09-18T23:55:22+08:00" }, { "file": "/var/log/nginx/infogap.access.log.8.gz", "bytes": 35549, "lines": 3063, "parsed_lines": 3063, "crawler_hits": 93, "window_start": "2026-09-17T00:02:26+08:00", "window_end": "2026-09-17T23:46:14+08:00" }, { "file": "/var/log/nginx/infogap.access.log.9.gz", "bytes": 25513, "lines": 2396, "parsed_lines": 2396, "crawler_hits": 4, "window_start": "2026-09-16T00:00:14+08:00", "window_end": "2026-09-16T23:54:04+08:00" } ], "errors": [], "by_day": { "2026-09-12": { "requests": 2119, "ai_hits": 682, "search_hits": 0, "by_ua": { "ChatGPT-User": 95, "OAI-SearchBot": 94, "Amazonbot": 89, "Perplexity-User": 88, "GPTBot": 73, "meta-externalagent": 55, "Bytespider": 52, "ClaudeBot": 47, "PerplexityBot": 45, "Applebot": 44 } }, "2026-09-13": { "requests": 2379, "ai_hits": 362, "search_hits": 0, "by_ua": { "GPTBot": 55, "ChatGPT-User": 50, "OAI-SearchBot": 50, "Amazonbot": 49, "Perplexity-User": 43, "PerplexityBot": 29, "Bytespider": 24, "Applebot": 21, "meta-externalagent": 21, "ClaudeBot": 20 } }, "2026-09-14": { "requests": 5151, "ai_hits": 2, "search_hits": 3, "by_ua": { "Bingbot": 3, "GPTBot": 1, "OAI-SearchBot": 1 } }, "2026-09-15": { "requests": 2466, "ai_hits": 1, "search_hits": 0, "by_ua": { "OAI-SearchBot": 1 } }, "2026-09-16": { "requests": 3280, "ai_hits": 1, "search_hits": 3, "by_ua": { "Bingbot": 2, "OAI-SearchBot": 1, "Googlebot": 1 } }, "2026-09-17": { "requests": 4041, "ai_hits": 88, "search_hits": 5, "by_ua": { "GPTBot": 87, "Bingbot": 5, "OAI-SearchBot": 1 } }, "2026-09-18": { "requests": 2995, "ai_hits": 38, "search_hits": 72, "by_ua": { "Googlebot": 72, "GPTBot": 32, "Applebot": 4, "OAI-SearchBot": 2 } }, "2026-09-19": { "requests": 11458, "ai_hits": 3, "search_hits": 1222, "by_ua": { "Googlebot": 1208, "Bingbot": 14, "OAI-SearchBot": 2, "GPTBot": 1 } }, "2026-09-20": { "requests": 19529, "ai_hits": 1, "search_hits": 11, "by_ua": { "Bingbot": 11, "OAI-SearchBot": 1 } }, "2026-09-21": { "requests": 11928, "ai_hits": 8, "search_hits": 22, "by_ua": { "Bingbot": 22, "GPTBot": 5, "OAI-SearchBot": 3 } }, "2026-09-22": { "requests": 9919, "ai_hits": 27, "search_hits": 204, "by_ua": { "Googlebot": 196, "GPTBot": 25, "Bingbot": 8, "OAI-SearchBot": 2 } }, "2026-09-23": { "requests": 10677, "ai_hits": 59, "search_hits": 650, "by_ua": { "Googlebot": 646, "GPTBot": 56, "Bingbot": 4, "OAI-SearchBot": 3 } }, "2026-09-24": { "requests": 9205, "ai_hits": 10, "search_hits": 630, "by_ua": { "Googlebot": 628, "OAI-SearchBot": 9, "Bingbot": 2, "ChatGPT-User": 1 } }, "2026-09-25": { "requests": 8075, "ai_hits": 440, "search_hits": 46, "by_ua": { "GPTBot": 436, "Googlebot": 45, "OAI-SearchBot": 2, "ChatGPT-User": 2, "Bingbot": 1 } } }, "by_ua": { "Googlebot": { "category": "search", "hits": 2796, "first_seen": "2026-09-16T19:31:17+08:00", "last_seen": "2026-09-25T14:15:30+08:00", "by_day": { "2026-09-16": 1, "2026-09-18": 72, "2026-09-19": 1208, "2026-09-22": 196, "2026-09-23": 646, "2026-09-24": 628, "2026-09-25": 45 }, "status": { "404": 2472, "200": 317, "301": 7 } }, "GPTBot": { "category": "ai", "hits": 771, "first_seen": "2026-09-12T12:08:13+08:00", "last_seen": "2026-09-25T10:38:11+08:00", "by_day": { "2026-09-12": 73, "2026-09-13": 55, "2026-09-14": 1, "2026-09-17": 87, "2026-09-18": 32, "2026-09-19": 1, "2026-09-21": 5, "2026-09-22": 25, "2026-09-23": 56, "2026-09-25": 436 }, "status": { "200": 641, "404": 130 } }, "OAI-SearchBot": { "category": "ai", "hits": 172, "first_seen": "2026-09-12T12:08:13+08:00", "last_seen": "2026-09-25T10:34:09+08:00", "by_day": { "2026-09-12": 94, "2026-09-13": 50, "2026-09-14": 1, "2026-09-15": 1, "2026-09-16": 1, "2026-09-17": 1, "2026-09-18": 2, "2026-09-19": 2, "2026-09-20": 1, "2026-09-21": 3, "2026-09-22": 2, "2026-09-23": 3, "2026-09-24": 9, "2026-09-25": 2 }, "status": { "404": 147, "200": 25 } }, "ChatGPT-User": { "category": "ai", "hits": 148, "first_seen": "2026-09-12T14:35:01+08:00", "last_seen": "2026-09-25T14:44:10+08:00", "by_day": { "2026-09-12": 95, "2026-09-13": 50, "2026-09-24": 1, "2026-09-25": 2 }, "status": { "404": 145, "200": 3 } }, "Amazonbot": { "category": "ai", "hits": 138, "first_seen": "2026-09-12T14:35:01+08:00", "last_seen": "2026-09-13T12:15:25+08:00", "by_day": { "2026-09-12": 89, "2026-09-13": 49 }, "status": { "404": 138 } }, "Perplexity-User": { "category": "ai", "hits": 131, "first_seen": "2026-09-12T14:35:06+08:00", "last_seen": "2026-09-13T12:15:25+08:00", "by_day": { "2026-09-12": 88, "2026-09-13": 43 }, "status": { "404": 130, "200": 1 } }, "meta-externalagent": { "category": "ai", "hits": 76, "first_seen": "2026-09-12T14:35:01+08:00", "last_seen": "2026-09-13T12:15:23+08:00", "by_day": { "2026-09-12": 55, "2026-09-13": 21 }, "status": { "404": 76 } }, "Bytespider": { "category": "ai", "hits": 76, "first_seen": "2026-09-12T14:35:04+08:00", "last_seen": "2026-09-13T12:15:25+08:00", "by_day": { "2026-09-12": 52, "2026-09-13": 24 }, "status": { "404": 76 } }, "PerplexityBot": { "category": "ai", "hits": 74, "first_seen": "2026-09-12T14:35:06+08:00", "last_seen": "2026-09-13T12:15:24+08:00", "by_day": { "2026-09-12": 45, "2026-09-13": 29 }, "status": { "404": 74 } }, "Bingbot": { "category": "search", "hits": 72, "first_seen": "2026-09-14T08:24:58+08:00", "last_seen": "2026-09-25T10:32:39+08:00", "by_day": { "2026-09-14": 3, "2026-09-16": 2, "2026-09-17": 5, "2026-09-19": 14, "2026-09-20": 11, "2026-09-21": 22, "2026-09-22": 8, "2026-09-23": 4, "2026-09-24": 2, "2026-09-25": 1 }, "status": { "200": 48, "404": 15, "301": 9 } }, "Applebot": { "category": "ai", "hits": 69, "first_seen": "2026-09-12T14:35:01+08:00", "last_seen": "2026-09-18T10:48:26+08:00", "by_day": { "2026-09-12": 44, "2026-09-13": 21, "2026-09-18": 4 }, "status": { "404": 65, "301": 2, "200": 2 } }, "ClaudeBot": { "category": "ai", "hits": 67, "first_seen": "2026-09-12T11:45:45+08:00", "last_seen": "2026-09-13T12:15:25+08:00", "by_day": { "2026-09-12": 47, "2026-09-13": 20 }, "status": { "404": 66, "200": 1 } } }, "top_urls": [ { "url": "/robots.txt", "hits": 47, "uas": { "OAI-SearchBot": 24, "Bingbot": 10, "Googlebot": 8, "Applebot": 4, "ClaudeBot": 1 } }, { "url": "/", "hits": 46, "uas": { "Googlebot": 29, "GPTBot": 10, "Bingbot": 4, "Amazonbot": 1, "OAI-SearchBot": 1, "ClaudeBot": 1 } }, { "url": "/.git/HEAD", "hits": 21, "uas": { "Googlebot": 18, "Applebot": 1, "ClaudeBot": 1, "GPTBot": 1 } }, { "url": "/.aws/credentials", "hits": 21, "uas": { "Googlebot": 18, "GPTBot": 1, "OAI-SearchBot": 1, "Bytespider": 1 } }, { "url": "/.travis.yml", "hits": 21, "uas": { "Googlebot": 18, "OAI-SearchBot": 2, "GPTBot": 1 } }, { "url": "/.gitlab-ci.yml", "hits": 21, "uas": { "Googlebot": 18, "Amazonbot": 1, "Bingbot": 1, "Bytespider": 1 } }, { "url": "/wp-config.php~", "hits": 20, "uas": { "Googlebot": 18, "OAI-SearchBot": 1, "ChatGPT-User": 1 } }, { "url": "/.git/config", "hits": 20, "uas": { "Googlebot": 18, "OAI-SearchBot": 1, "Perplexity-User": 1 } }, { "url": "/wp-config.php.bak", "hits": 20, "uas": { "Googlebot": 18, "ChatGPT-User": 1, "Applebot": 1 } }, { "url": "/backend/.env", "hits": 20, "uas": { "Googlebot": 17, "PerplexityBot": 1, "OAI-SearchBot": 1, "meta-externalagent": 1 } }, { "url": "/config.js", "hits": 20, "uas": { "Googlebot": 17, "OAI-SearchBot": 1, "Applebot": 1, "Bytespider": 1 } }, { "url": "/settings.py", "hits": 19, "uas": { "Googlebot": 18, "Amazonbot": 1 } }, { "url": "/.docker/.env", "hits": 19, "uas": { "Googlebot": 16, "Applebot": 1, "OAI-SearchBot": 1, "Perplexity-User": 1 } }, { "url": "/.env.local", "hits": 19, "uas": { "Googlebot": 17, "ClaudeBot": 1, "meta-externalagent": 1 } }, { "url": "/docker-compose.yaml", "hits": 19, "uas": { "Googlebot": 18, "Bytespider": 1 } }, { "url": "/.env.test", "hits": 19, "uas": { "Googlebot": 16, "ClaudeBot": 1, "Bingbot": 1, "ChatGPT-User": 1 } }, { "url": "/wp-config.php.old", "hits": 19, "uas": { "Googlebot": 18, "Applebot": 1 } }, { "url": "/config/.env", "hits": 19, "uas": { "Googlebot": 16, "PerplexityBot": 1, "Bytespider": 1, "ChatGPT-User": 1 } }, { "url": "/test.php", "hits": 19, "uas": { "Googlebot": 16, "Amazonbot": 1, "ChatGPT-User": 1, "Perplexity-User": 1 } }, { "url": "/web/.env", "hits": 19, "uas": { "Googlebot": 16, "meta-externalagent": 1, "Bingbot": 1, "ChatGPT-User": 1 } }, { "url": "/config/application.yml", "hits": 19, "uas": { "Googlebot": 16, "OAI-SearchBot": 1, "Amazonbot": 1, "GPTBot": 1 } }, { "url": "/.env.json", "hits": 19, "uas": { "Googlebot": 16, "Bytespider": 2, "Applebot": 1 } }, { "url": "/config/master.key", "hits": 19, "uas": { "Googlebot": 16, "GPTBot": 2, "Bytespider": 1 } }, { "url": "/.aws/config", "hits": 19, "uas": { "Googlebot": 18, "ChatGPT-User": 1 } }, { "url": "/.ssh/id_dsa", "hits": 19, "uas": { "Googlebot": 16, "Applebot": 1, "ClaudeBot": 1, "ChatGPT-User": 1 } }, { "url": "/config.json", "hits": 19, "uas": { "Googlebot": 17, "Perplexity-User": 1, "ChatGPT-User": 1 } }, { "url": "/docker-compose.yml", "hits": 19, "uas": { "Googlebot": 18, "Perplexity-User": 1 } }, { "url": "/local_settings.py", "hits": 18, "uas": { "Googlebot": 18 } }, { "url": "/config/app.php", "hits": 18, "uas": { "Googlebot": 18 } }, { "url": "/.env.development", "hits": 18, "uas": { "Googlebot": 16, "Perplexity-User": 2 } } ], "status_mix_all": { "200": 66499, "404": 32591, "other": 4132, "other_detail": { "201": 48, "206": 151, "301": 2033, "304": 49, "400": 1186, "401": 466, "403": 22, "405": 92, "410": 1, "413": 9, "429": 26, "499": 38, "501": 11 } }, "status_mix_ai": { "200": 673, "404": 1047, "other": 2, "other_detail": { "301": 2 } }, "ai_404": { "count": 1047, "by_ua": { "OAI-SearchBot": 147, "ChatGPT-User": 145, "Amazonbot": 138, "GPTBot": 130, "Perplexity-User": 130, "meta-externalagent": 76, "Bytespider": 76, "PerplexityBot": 74, "ClaudeBot": 66, "Applebot": 65 }, "top_urls": [ { "url": "/robots.txt", "hits": 6 }, { "url": "/fetch", "hits": 5 }, { "url": "/media../etc/passwd", "hits": 3 }, { "url": "/.aws/credentials", "hits": 3 }, { "url": "/static../.azure/credentials", "hits": 3 }, { "url": "/@fs/root/.config/gcloud/application_default_credentials.json?raw??", "hits": 3 }, { "url": "/aws/credentials", "hits": 3 }, { "url": "/.git-credentials", "hits": 3 }, { "url": "/config.js", "hits": 3 }, { "url": "/.git/HEAD", "hits": 3 }, { "url": "/config/.env", "hits": 3 }, { "url": "/.env.docker", "hits": 3 }, { "url": "/backend/.env", "hits": 3 }, { "url": "/.env.gcp", "hits": 3 }, { "url": "/fetch?url=http%3A%2F%2F169.254.169.254%2Flatest%2Fmeta-data%2Fiam%2Finfo", "hits": 3 }, { "url": "/fetch?uri=http%3A%2F%2F169.254.169.254%2Flatest%2Fmeta-data%2F", "hits": 3 }, { "url": "/graphql", "hits": 3 }, { "url": "/v1/graphql", "hits": 3 }, { "url": "/graphql/console", "hits": 3 }, { "url": "/fetch?path=http%3A%2F%2F169.254.169.254%2Flatest%2Fmeta-data%2Fiam%2Finfo", "hits": 3 }, { "url": "/.aws/credentials.bak", "hits": 3 }, { "url": "/app/.aws/credentials", "hits": 3 }, { "url": "/fetch?path=http%3A%2F%2F169.254.169.254%2Flatest%2Fmeta-data%2Fiam%2Fsecurity-credentials%2F", "hits": 3 }, { "url": "/management", "hits": 3 }, { "url": "/_next/static/../.env", "hits": 3 }, { "url": "/_next/../.aws/credentials", "hits": 3 }, { "url": "/@fs/home/node/.aws/credentials?raw??", "hits": 3 }, { "url": "/@fs/root/.aws/config?raw??", "hits": 3 }, { "url": "/@fs/root/.azure/credentials?raw??", "hits": 3 }, { "url": "/@fs/home/node/.config/gcloud/application_default_credentials.json?raw??", "hits": 3 } ] } }, "shengminglian.com.cn": { "generated_at": "2026-09-25T16:11:55.094664+08:00", "label": "CN", "hostname": "hcss-ecs-a99a", "log_dir": "/var/log/nginx", "log_bases": [ "infogap.access.log", "access.log" ], "days_filter": 30, "window": { "start": "2026-09-17T00:11:02+08:00", "end": "2026-09-25T16:11:51+08:00", "reference_day": "2026-09-25", "cutoff_day": "2026-08-27" }, "total_lines": 190378, "parsed_lines": 190378, "ai_hits": 2337, "search_hits": 629, "files_scanned": [ { "file": "/var/log/nginx/access.log", "bytes": 1061588, "lines": 8922, "parsed_lines": 8922, "crawler_hits": 109, "window_start": "2026-09-25T00:00:01+08:00", "window_end": "2026-09-25T16:11:51+08:00" }, { "file": "/var/log/nginx/access.log.1", "bytes": 1801375, "lines": 13851, "parsed_lines": 13851, "crawler_hits": 52, "window_start": "2026-09-24T00:00:02+08:00", "window_end": "2026-09-25T00:00:01+08:00" }, { "file": "/var/log/nginx/access.log.2.gz", "bytes": 174002, "lines": 17903, "parsed_lines": 17903, "crawler_hits": 343, "window_start": "2026-09-23T00:00:02+08:00", "window_end": "2026-09-24T00:00:02+08:00" }, { "file": "/var/log/nginx/access.log.3.gz", "bytes": 222255, "lines": 20568, "parsed_lines": 20568, "crawler_hits": 1193, "window_start": "2026-09-22T00:00:02+08:00", "window_end": "2026-09-22T23:59:33+08:00" }, { "file": "/var/log/nginx/access.log.4.gz", "bytes": 420233, "lines": 34874, "parsed_lines": 34874, "crawler_hits": 787, "window_start": "2026-09-21T00:00:03+08:00", "window_end": "2026-09-22T00:00:02+08:00" }, { "file": "/var/log/nginx/access.log.5.gz", "bytes": 620133, "lines": 48981, "parsed_lines": 48981, "crawler_hits": 14, "window_start": "2026-09-20T00:00:12+08:00", "window_end": "2026-09-21T00:00:03+08:00" }, { "file": "/var/log/nginx/access.log.6.gz", "bytes": 420662, "lines": 36209, "parsed_lines": 36209, "crawler_hits": 389, "window_start": "2026-09-19T00:00:02+08:00", "window_end": "2026-09-20T00:00:02+08:00" }, { "file": "/var/log/nginx/access.log.7.gz", "bytes": 61511, "lines": 7285, "parsed_lines": 7285, "crawler_hits": 16, "window_start": "2026-09-18T00:03:56+08:00", "window_end": "2026-09-19T00:00:02+08:00" }, { "file": "/var/log/nginx/access.log.8.gz", "bytes": 28769, "lines": 1785, "parsed_lines": 1785, "crawler_hits": 63, "window_start": "2026-09-17T00:11:02+08:00", "window_end": "2026-09-17T23:59:43+08:00" } ], "errors": [], "by_day": { "2026-09-17": { "requests": 1785, "ai_hits": 49, "search_hits": 14, "by_ua": { "GPTBot": 45, "Bingbot": 12, "OAI-SearchBot": 4, "Googlebot": 2 } }, "2026-09-18": { "requests": 7273, "ai_hits": 7, "search_hits": 9, "by_ua": { "Bingbot": 7, "Applebot": 3, "OAI-SearchBot": 2, "GPTBot": 2, "Googlebot": 2 } }, "2026-09-19": { "requests": 36195, "ai_hits": 14, "search_hits": 375, "by_ua": { "Googlebot": 371, "Applebot": 5, "OAI-SearchBot": 4, "Bingbot": 4, "GPTBot": 3, "Bytespider": 2 } }, "2026-09-20": { "requests": 48997, "ai_hits": 8, "search_hits": 6, "by_ua": { "Bingbot": 6, "GPTBot": 5, "OAI-SearchBot": 3 } }, "2026-09-21": { "requests": 34877, "ai_hits": 765, "search_hits": 22, "by_ua": { "GPTBot": 755, "Bingbot": 15, "OAI-SearchBot": 10, "Googlebot": 7 } }, "2026-09-22": { "requests": 20575, "ai_hits": 1106, "search_hits": 87, "by_ua": { "GPTBot": 901, "Googlebot": 86, "ClaudeBot": 35, "OAI-SearchBot": 28, "CCBot": 17, "meta-externalagent": 17, "Bytespider": 14, "MistralAI-User": 14, "PerplexityBot": 13, "Perplexity-User": 13, "Amazonbot": 12, "ChatGPT-User": 10, "Applebot": 10, "YouBot": 8, "cohere-ai": 7, "DuckAssistBot": 7, "Bingbot": 1 } }, "2026-09-23": { "requests": 17900, "ai_hits": 288, "search_hits": 55, "by_ua": { "Googlebot": 42, "ClaudeBot": 37, "OAI-SearchBot": 24, "GPTBot": 24, "meta-externalagent": 20, "cohere-ai": 19, "MistralAI-User": 18, "PerplexityBot": 18, "YouBot": 18, "Applebot": 17, "Amazonbot": 17, "Bytespider": 16, "DuckAssistBot": 16, "ChatGPT-User": 15, "CCBot": 15, "Perplexity-User": 14, "Bingbot": 13 } }, "2026-09-24": { "requests": 13849, "ai_hits": 19, "search_hits": 33, "by_ua": { "Bingbot": 19, "Googlebot": 14, "ChatGPT-User": 8, "meta-externalagent": 6, "OAI-SearchBot": 3, "GPTBot": 2 } }, "2026-09-25": { "requests": 8927, "ai_hits": 81, "search_hits": 28, "by_ua": { "GPTBot": 77, "Googlebot": 24, "Bingbot": 4, "OAI-SearchBot": 4 } } }, "by_ua": { "GPTBot": { "category": "ai", "hits": 1814, "first_seen": "2026-09-17T08:57:00+08:00", "last_seen": "2026-09-25T13:54:16+08:00", "by_day": { "2026-09-17": 45, "2026-09-18": 2, "2026-09-19": 3, "2026-09-20": 5, "2026-09-21": 755, "2026-09-22": 901, "2026-09-23": 24, "2026-09-24": 2, "2026-09-25": 77 }, "status": { "200": 1731, "404": 71, "403": 11, "301": 1 } }, "Googlebot": { "category": "search", "hits": 548, "first_seen": "2026-09-17T11:33:38+08:00", "last_seen": "2026-09-25T16:07:18+08:00", "by_day": { "2026-09-17": 2, "2026-09-18": 2, "2026-09-19": 371, "2026-09-21": 7, "2026-09-22": 86, "2026-09-23": 42, "2026-09-24": 14, "2026-09-25": 24 }, "status": { "301": 390, "200": 135, "404": 14, "403": 9 } }, "OAI-SearchBot": { "category": "ai", "hits": 82, "first_seen": "2026-09-17T08:57:00+08:00", "last_seen": "2026-09-25T13:53:59+08:00", "by_day": { "2026-09-17": 4, "2026-09-18": 2, "2026-09-19": 4, "2026-09-20": 3, "2026-09-21": 10, "2026-09-22": 28, "2026-09-23": 24, "2026-09-24": 3, "2026-09-25": 4 }, "status": { "200": 38, "404": 23, "403": 14, "301": 7 } }, "Bingbot": { "category": "search", "hits": 81, "first_seen": "2026-09-17T01:04:15+08:00", "last_seen": "2026-09-25T13:22:26+08:00", "by_day": { "2026-09-17": 12, "2026-09-18": 7, "2026-09-19": 4, "2026-09-20": 6, "2026-09-21": 15, "2026-09-22": 1, "2026-09-23": 13, "2026-09-24": 19, "2026-09-25": 4 }, "status": { "200": 78, "404": 2, "304": 1 } }, "ClaudeBot": { "category": "ai", "hits": 72, "first_seen": "2026-09-22T17:45:05+08:00", "last_seen": "2026-09-23T02:18:06+08:00", "by_day": { "2026-09-22": 35, "2026-09-23": 37 }, "status": { "404": 49, "403": 22, "405": 1 } }, "meta-externalagent": { "category": "ai", "hits": 43, "first_seen": "2026-09-22T17:45:05+08:00", "last_seen": "2026-09-24T16:35:01+08:00", "by_day": { "2026-09-22": 17, "2026-09-23": 20, "2026-09-24": 6 }, "status": { "404": 23, "403": 16, "200": 4 } }, "Applebot": { "category": "ai", "hits": 35, "first_seen": "2026-09-18T11:53:18+08:00", "last_seen": "2026-09-23T02:18:06+08:00", "by_day": { "2026-09-18": 3, "2026-09-19": 5, "2026-09-22": 10, "2026-09-23": 17 }, "status": { "404": 18, "403": 9, "200": 8 } }, "ChatGPT-User": { "category": "ai", "hits": 33, "first_seen": "2026-09-22T17:45:05+08:00", "last_seen": "2026-09-24T01:19:47+08:00", "by_day": { "2026-09-22": 10, "2026-09-23": 15, "2026-09-24": 8 }, "status": { "404": 20, "403": 9, "301": 4 } }, "Bytespider": { "category": "ai", "hits": 32, "first_seen": "2026-09-19T23:52:12+08:00", "last_seen": "2026-09-23T02:17:59+08:00", "by_day": { "2026-09-19": 2, "2026-09-22": 14, "2026-09-23": 16 }, "status": { "404": 18, "403": 11, "200": 2, "405": 1 } }, "MistralAI-User": { "category": "ai", "hits": 32, "first_seen": "2026-09-22T17:45:05+08:00", "last_seen": "2026-09-23T02:18:05+08:00", "by_day": { "2026-09-22": 14, "2026-09-23": 18 }, "status": { "404": 19, "403": 13 } }, "CCBot": { "category": "ai", "hits": 32, "first_seen": "2026-09-22T17:45:04+08:00", "last_seen": "2026-09-23T02:18:02+08:00", "by_day": { "2026-09-22": 17, "2026-09-23": 15 }, "status": { "404": 23, "403": 9 } }, "PerplexityBot": { "category": "ai", "hits": 31, "first_seen": "2026-09-22T17:45:05+08:00", "last_seen": "2026-09-23T02:18:02+08:00", "by_day": { "2026-09-22": 13, "2026-09-23": 18 }, "status": { "404": 22, "403": 9 } }, "Amazonbot": { "category": "ai", "hits": 29, "first_seen": "2026-09-22T17:45:06+08:00", "last_seen": "2026-09-23T02:18:01+08:00", "by_day": { "2026-09-22": 12, "2026-09-23": 17 }, "status": { "404": 23, "403": 5, "301": 1 } }, "Perplexity-User": { "category": "ai", "hits": 27, "first_seen": "2026-09-22T17:45:06+08:00", "last_seen": "2026-09-23T02:18:00+08:00", "by_day": { "2026-09-22": 13, "2026-09-23": 14 }, "status": { "404": 21, "403": 6 } }, "cohere-ai": { "category": "ai", "hits": 26, "first_seen": "2026-09-22T17:45:06+08:00", "last_seen": "2026-09-23T02:18:01+08:00", "by_day": { "2026-09-22": 7, "2026-09-23": 19 }, "status": { "404": 19, "403": 6, "405": 1 } }, "YouBot": { "category": "ai", "hits": 26, "first_seen": "2026-09-22T17:45:06+08:00", "last_seen": "2026-09-23T02:17:59+08:00", "by_day": { "2026-09-22": 8, "2026-09-23": 18 }, "status": { "404": 17, "403": 9 } }, "DuckAssistBot": { "category": "ai", "hits": 23, "first_seen": "2026-09-22T17:45:06+08:00", "last_seen": "2026-09-23T02:18:01+08:00", "by_day": { "2026-09-22": 7, "2026-09-23": 16 }, "status": { "404": 18, "403": 5 } } }, "top_urls": [ { "url": "/robots.txt", "hits": 94, "uas": { "OAI-SearchBot": 44, "Googlebot": 37, "Bingbot": 9, "Applebot": 2, "Bytespider": 2 } }, { "url": "/", "hits": 28, "uas": { "GPTBot": 12, "Googlebot": 9, "Bingbot": 6, "Bytespider": 1 } }, { "url": "/sitemap.xml", "hits": 17, "uas": { "GPTBot": 17 } }, { "url": "/echarts.min.js", "hits": 12, "uas": { "GPTBot": 4, "Googlebot": 4, "Applebot": 3, "Bingbot": 1 } }, { "url": "/daily_keywords.html", "hits": 9, "uas": { "Bingbot": 5, "GPTBot": 4 } }, { "url": "/data/industry_insights.json", "hits": 7, "uas": { "Googlebot": 6, "Bingbot": 1 } }, { "url": "/algo.html?type=anti_algo", "hits": 6, "uas": { "GPTBot": 4, "Bingbot": 1, "Applebot": 1 } }, { "url": "/seo_pages/%E7%BB%BC%E5%90%88%E8%AF%8A%E6%89%80.html", "hits": 6, "uas": { "GPTBot": 4, "Googlebot": 2 } }, { "url": "/seo_pages/%E9%85%92%E5%BA%97.html", "hits": 6, "uas": { "GPTBot": 4, "Googlebot": 2 } }, { "url": "/seo_pages/%E5%92%96%E5%95%A1%E5%BA%97.html", "hits": 6, "uas": { "GPTBot": 4, "meta-externalagent": 1, "Googlebot": 1 } }, { "url": "/seo_pages/%E4%BE%BF%E5%88%A9%E5%BA%97.html", "hits": 6, "uas": { "GPTBot": 4, "meta-externalagent": 1, "Googlebot": 1 } }, { "url": "/api/config", "hits": 6, "uas": { "Perplexity-User": 2, "Amazonbot": 1, "Googlebot": 1, "PerplexityBot": 1, "ChatGPT-User": 1 } }, { "url": "/seo_pages/%E5%81%A5%E8%BA%AB%E6%88%BF.html", "hits": 6, "uas": { "GPTBot": 4, "meta-externalagent": 1, "Googlebot": 1 } }, { "url": "/article.html?file=wanlian_article.md", "hits": 5, "uas": { "GPTBot": 4, "Googlebot": 1 } }, { "url": "/article.html?file=capital_eval_article.md", "hits": 5, "uas": { "GPTBot": 4, "Googlebot": 1 } }, { "url": "/algo/_viz/viz.js", "hits": 5, "uas": { "Googlebot": 3, "GPTBot": 2 } }, { "url": "/algo/_viz/viz.css", "hits": 5, "uas": { "Googlebot": 3, "GPTBot": 2 } }, { "url": "/more/", "hits": 5, "uas": { "GPTBot": 4, "Googlebot": 1 } }, { "url": "/secrets.json", "hits": 5, "uas": { "ChatGPT-User": 2, "ClaudeBot": 2, "Googlebot": 1 } }, { "url": "/topics/jia-zheng-gong-si-zen-me-kai.html", "hits": 5, "uas": { "GPTBot": 4, "Bingbot": 1 } }, { "url": "/seo_pages/", "hits": 5, "uas": { "GPTBot": 4, "Googlebot": 1 } }, { "url": "/seo_pages/%E7%81%AB%E9%94%85.html", "hits": 5, "uas": { "GPTBot": 4, "Googlebot": 1 } }, { "url": "/seo_pages/KTV.html", "hits": 5, "uas": { "GPTBot": 4, "Googlebot": 1 } }, { "url": "/seo_pages/%E7%8E%A9%E5%85%B7%E5%BA%97.html", "hits": 5, "uas": { "GPTBot": 4, "Googlebot": 1 } }, { "url": "/seo_pages/%E5%BF%AB%E9%80%92%E9%A9%BF%E7%AB%99.html", "hits": 5, "uas": { "GPTBot": 4, "Googlebot": 1 } }, { "url": "/seo_pages/%E7%94%B5%E5%8A%A8%E8%BD%A6%E8%A1%8C.html", "hits": 5, "uas": { "GPTBot": 4, "Googlebot": 1 } }, { "url": "/seo_pages/%E5%AE%A0%E7%89%A9%E7%BE%8E%E5%AE%B9%E5%BA%97.html", "hits": 5, "uas": { "GPTBot": 4, "Googlebot": 1 } }, { "url": "/seo_pages/%E7%BA%B9%E8%BA%AB%E5%BA%97%28%E5%88%BA%E9%9D%92%E8%89%BA%E6%9C%AF%29.html", "hits": 5, "uas": { "GPTBot": 4, "Googlebot": 1 } }, { "url": "/seo_pages/%E6%B4%97%E8%BD%A6%E8%A1%8C%28%E6%B4%97%E8%BD%A6_%E6%B1%BD%E8%BD%A6%E7%BE%8E%E5%AE%B9%29.html", "hits": 5, "uas": { "GPTBot": 4, "Googlebot": 1 } }, { "url": "/platform.html", "hits": 5, "uas": { "GPTBot": 4, "Bingbot": 1 } } ], "status_mix_all": { "200": 163525, "404": 16457, "other": 10396, "other_detail": { "101": 13, "206": 75, "301": 4670, "302": 10, "304": 27, "400": 1281, "401": 60, "403": 3837, "405": 140, "408": 2, "409": 4, "413": 2, "415": 1, "422": 17, "429": 63, "499": 47, "500": 57, "502": 90 } }, "status_mix_ai": { "200": 1783, "404": 384, "other": 170, "other_detail": { "301": 13, "403": 154, "405": 3 } }, "ai_404": { "count": 384, "by_ua": { "GPTBot": 71, "ClaudeBot": 49, "meta-externalagent": 23, "Amazonbot": 23, "OAI-SearchBot": 23, "CCBot": 23, "PerplexityBot": 22, "Perplexity-User": 21, "ChatGPT-User": 20, "cohere-ai": 19, "MistralAI-User": 19, "Applebot": 18, "Bytespider": 18, "DuckAssistBot": 18, "YouBot": 17 }, "top_urls": [ { "url": "/api/config", "hits": 5 }, { "url": "/geo/lib/leaflet.js.map", "hits": 4 }, { "url": "/geo/lib/images/layers.png", "hits": 4 }, { "url": "/geo/lib/images/layers-2x.png", "hits": 4 }, { "url": "/secrets.json", "hits": 3 }, { "url": "/seo_pages/null", "hits": 3 }, { "url": "/configuration.php.bak", "hits": 3 }, { "url": "/wp-config.php~", "hits": 3 }, { "url": "/bootstrap.properties", "hits": 3 }, { "url": "/actuator/loggers", "hits": 3 }, { "url": "/config/database.yml", "hits": 3 }, { "url": "/local.settings.json", "hits": 3 }, { "url": "/config/storage.yml", "hits": 3 }, { "url": "/settings.py", "hits": 3 }, { "url": "/console", "hits": 3 }, { "url": "/openapi.json", "hits": 3 }, { "url": "/sendgrid.env", "hits": 3 }, { "url": "/settings/_payload.json", "hits": 3 }, { "url": "/_image?href=/proc/self/environ", "hits": 3 }, { "url": "/metrics", "hits": 3 }, { "url": "/aws-exports.js", "hits": 3 }, { "url": "/aws-config.js", "hits": 3 }, { "url": "/awsConfig.js", "hits": 3 }, { "url": "/debug/vars", "hits": 3 }, { "url": "/api/fs/exec", "hits": 3 }, { "url": "/read-document", "hits": 3 }, { "url": "/api/designer/v1/file-content", "hits": 3 }, { "url": "/api/templates/preview", "hits": 3 }, { "url": "/topics/peixun-jigou-pai-ming-qianshi.html", "hits": 3 }, { "url": "/topics/pei-xun-ji-gou-pai-ming-qian-shi.html", "hits": 3 } ] } } } }