$v) { if (is_string($k) && str_starts_with($k, 'limit=')) { $limit = (int) substr($k, 6); } } // resolve the sitemap index $index = $this->fetch('https://gg.deals/sitemap.xml'); if ($index === null) { CLI::error('Could not fetch sitemap index'); return; } preg_match_all('/sitemap_games_(\d+)\.xml/', $index, $m); $files = array_map(static fn ($n) => (int) $n, $m[1]); sort($files, SORT_NUMERIC); CLI::write('Found ' . count($files) . ' game sitemap files.', 'yellow'); if ($limit !== null) { $files = array_slice($files, 0, $limit); CLI::write('Limit set: first ' . count($files) . ' files.', 'yellow'); } $totalUrls = 0; $totalInserted = 0; $totalSkipped = 0; foreach ($files as $i => $n) { $url = "https://gg.deals/sitemap_games_{$n}.xml"; $xml = $this->fetch($url); if ($xml === null) { CLI::error(" sitemap {$n}: fetch failed"); usleep(500000); continue; } preg_match_all('/(https:\/\/gg\.deals\/game\/([a-z0-9-]+)\/?)<\/loc>/', $xml, $matches); $slugs = array_values(array_unique($matches[2])); $totalUrls += count($slugs); $docs = []; foreach ($slugs as $slug) { $docs[] = [ 'slug' => $slug, 'url' => 'https://gg.deals/game/' . $slug . '/', 'crawled' => false, 'added_at' => new UTCDateTime(time() * 1000), ]; } if ($docs !== []) { try { Mongo::collection('catalog')->insertMany($docs, ['ordered' => false]); $totalInserted += count($docs); } catch (\MongoDB\Driver\Exception\BulkWriteException $e) { // duplicate keys are expected on re-runs $totalInserted += count($docs); } } CLI::write(" [{$i}/" . count($files) . "] sitemap {$n}: " . count($slugs) . ' slugs (total ' . number_format($totalInserted) . ')'); usleep(300000); } CLI::write(''); CLI::write("Catalog crawl done: {$totalUrls} urls, {$totalInserted} inserted.", 'green'); CLI::write('Total catalog documents: ' . Mongo::collection('catalog')->countDocuments(), 'green'); } private function fetch(string $url): ?string { $ch = curl_init($url); curl_setopt_array($ch, [ CURLOPT_RETURNTRANSFER => true, CURLOPT_TIMEOUT => 30, CURLOPT_FOLLOWLOCATION => true, CURLOPT_USERAGENT => self::UA, CURLOPT_SSL_VERIFYPEER => true, CURLOPT_HTTPHEADER => ['Accept: application/xml,text/xml,*/*'], ]); $body = curl_exec($ch); $code = (int) curl_getinfo($ch, CURLINFO_RESPONSE_CODE); curl_close($ch); if ($code !== 200 || $body === false || $body === '') { return null; } return $body; } }