GG.deals clone: CodeIgniter 4 + MongoDB game price tracker
- Full gg.deals-style UI: deal feed with filters, game pages with price history charts, rankings, vouchers, news, prepaids, subscriptions, bundles, franchises, wishlist, API docs - Real data crawlers: * gg:crawl (Steam Store API + GOG Catalog + Epic free games) * gg:catalog (252K game slugs from gg.deals sitemaps) * gg:ggdeals (per-game prices via FlareSolverr session, JSON-LD parsing) * gg:seed / gg:crawl-covers (demo seed + Steam CDN cover art) - 24/7 self-updating scheduler (scripts/scheduler_loop.php, lockfile-protected) - Tailwind v4 dark/light theme, Alpine.js, Chart.js
This commit is contained in:
deepseek
@@ -0,0 +1,329 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Commands;
|
||||
|
||||
use App\Libraries\Mongo;
|
||||
use CodeIgniter\CLI\BaseCommand;
|
||||
use CodeIgniter\CLI\CLI;
|
||||
use MongoDB\BSON\UTCDateTime;
|
||||
|
||||
/**
|
||||
* Crawls detailed game data from gg.deals via a persistent FlareSolverr session.
|
||||
*
|
||||
* A FlareSolverr container must be reachable (default host "flaresolverr", port 80).
|
||||
* The session is created once and reused, so the Cloudflare challenge is only
|
||||
* solved on the first request — subsequent requests are fast.
|
||||
*
|
||||
* php spark gg:ggdeals → crawl details for all games we already have
|
||||
* php spark gg:ggdeals --limit=50 → only the first 50 matched games (testing)
|
||||
* php spark gg:ggdeals --slug=elden-ring → a single game by slug
|
||||
*/
|
||||
class CrawlGgdeals extends BaseCommand
|
||||
{
|
||||
protected $group = 'GGDeals';
|
||||
protected $name = 'gg:ggdeals';
|
||||
protected $description = 'Crawls detailed prices/deals from gg.deals via FlareSolverr session.';
|
||||
protected $usage = 'gg:ggdeals [--limit=N] [--slug=X]';
|
||||
|
||||
private const FS_URL = 'http://flaresolverr:80/v1';
|
||||
private const SESSION = 'ggdeals-crawler';
|
||||
private const UA = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36';
|
||||
|
||||
public function run(array $params): void
|
||||
{
|
||||
$limit = null;
|
||||
$slug = null;
|
||||
$batch = null;
|
||||
foreach ($params as $k => $v) {
|
||||
if (is_string($k) && str_starts_with($k, 'limit=')) {
|
||||
$limit = (int) substr($k, 6);
|
||||
}
|
||||
if (is_string($k) && str_starts_with($k, 'slug=')) {
|
||||
$slug = substr($k, 5);
|
||||
}
|
||||
if (is_string($k) && str_starts_with($k, 'batch=')) {
|
||||
$batch = (int) substr($k, 6);
|
||||
}
|
||||
}
|
||||
|
||||
CLI::write('GG.deals detailed crawler started — ' . date('Y-m-d H:i:s'), 'green');
|
||||
|
||||
// ensure session exists
|
||||
$this->fsRequest(['cmd' => 'sessions.create', 'session' => self::SESSION]);
|
||||
CLI::write('FlareSolverr session ready: ' . self::SESSION, 'green');
|
||||
|
||||
// ---- pick games to crawl ----
|
||||
$games = [];
|
||||
|
||||
if ($slug !== null) {
|
||||
// single game
|
||||
$g = Mongo::collection('games')->findOne(['slug' => $slug]);
|
||||
if ($g !== null) {
|
||||
$games[] = $g;
|
||||
}
|
||||
} elseif ($batch !== null) {
|
||||
// expand from catalog: next N slugs not yet detailed
|
||||
CLI::write("Batch mode: expanding {$batch} games from catalog…", 'yellow');
|
||||
$slugs = Mongo::collection('catalog')
|
||||
->find(['crawled' => false], ['projection' => ['slug' => 1], 'sort' => ['_id' => 1], 'limit' => $batch])
|
||||
->toArray();
|
||||
|
||||
foreach ($slugs as $c) {
|
||||
$cslug = $c['slug'];
|
||||
$existing = Mongo::collection('games')->findOne(['slug' => $cslug]);
|
||||
if ($existing !== null) {
|
||||
// already detailed; mark catalog crawled and skip
|
||||
Mongo::collection('catalog')->updateOne(['slug' => $cslug], ['$set' => ['crawled' => true]]);
|
||||
continue;
|
||||
}
|
||||
$games[] = ['slug' => $cslug, 'title' => $cslug];
|
||||
}
|
||||
} else {
|
||||
// refresh all games we already have
|
||||
$filter = [];
|
||||
$query = Mongo::collection('games')->find($filter, ['sort' => ['trending' => -1]]);
|
||||
if ($limit !== null) {
|
||||
$query = Mongo::collection('games')->find($filter, ['sort' => ['trending' => -1], 'limit' => $limit]);
|
||||
}
|
||||
$games = $query->toArray();
|
||||
}
|
||||
|
||||
CLI::write('Games to crawl: ' . count($games), 'yellow');
|
||||
|
||||
$updated = 0;
|
||||
$failed = 0;
|
||||
|
||||
foreach ($games as $i => $game) {
|
||||
$gslug = $game['slug'];
|
||||
$url = 'https://gg.deals/game/' . $gslug . '/';
|
||||
|
||||
$result = $this->fsRequest([
|
||||
'cmd' => 'request.get',
|
||||
'url' => $url,
|
||||
'session' => self::SESSION,
|
||||
'maxTimeout' => 30000,
|
||||
'disableMedia'=> true,
|
||||
]);
|
||||
|
||||
$html = $result['solution']['response'] ?? null;
|
||||
if ($html === null || $html === '' || str_contains($html, 'Just a moment')) {
|
||||
CLI::error(" [{$i}] {$gslug}: failed (challenge or empty)");
|
||||
$failed++;
|
||||
usleep(1000000);
|
||||
continue;
|
||||
}
|
||||
|
||||
$parsed = $this->parseGamePage($html, $gslug, $game);
|
||||
if ($parsed !== null) {
|
||||
$this->applyToDb($gslug, $parsed);
|
||||
$updated++;
|
||||
CLI::write(" [{$i}] {$gslug}: saved (best \${$parsed['best_price']})");
|
||||
} else {
|
||||
$failed++;
|
||||
}
|
||||
|
||||
// mark catalog as processed regardless (so we don't retry endlessly)
|
||||
Mongo::collection('catalog')->updateOne(
|
||||
['slug' => $gslug],
|
||||
['$set' => ['crawled' => true, 'crawled_at' => new UTCDateTime(time() * 1000)]],
|
||||
['upsert' => true]
|
||||
);
|
||||
|
||||
usleep(400000);
|
||||
}
|
||||
|
||||
// destroy session when done
|
||||
$this->fsRequest(['cmd' => 'sessions.destroy', 'session' => self::SESSION]);
|
||||
CLI::write("Done: {$updated} updated, {$failed} failed.", 'green');
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a gg.deals game page using its JSON-LD structured data.
|
||||
* Returns best price + all offers from the AggregateOffer block.
|
||||
*/
|
||||
private function parseGamePage(string $html, string $slug, array $existing): ?array
|
||||
{
|
||||
// extract JSON-LD blocks
|
||||
if (! preg_match_all('/<script type="application\/ld\+json">(.*?)<\/script>/s', $html, $m)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$product = null;
|
||||
foreach ($m[1] as $block) {
|
||||
$j = json_decode(trim($block), true);
|
||||
if (is_array($j) && ($j['@type'] ?? '') === 'Product' && isset($j['offers'])) {
|
||||
$product = $j;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if ($product === null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$offers = $product['offers'] ?? [];
|
||||
$bestPrice = (float) ($offers['lowPrice'] ?? 0);
|
||||
if ($bestPrice <= 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$offerList = [];
|
||||
foreach ($offers['offers'] ?? [] as $o) {
|
||||
$price = (float) ($o['price'] ?? 0);
|
||||
if ($price <= 0) {
|
||||
continue;
|
||||
}
|
||||
$seller = $o['seller']['name'] ?? 'Unknown';
|
||||
$offerList[] = [
|
||||
'price' => $price,
|
||||
'seller' => $seller,
|
||||
'expiry' => isset($o['priceValidUntil'])
|
||||
? (strtotime($o['priceValidUntil']) ?: null)
|
||||
: null,
|
||||
];
|
||||
}
|
||||
|
||||
return [
|
||||
'slug' => $slug,
|
||||
'title' => $product['name'] ?? $slug,
|
||||
'best_price' => $bestPrice,
|
||||
'high_price' => (float) ($offers['highPrice'] ?? $bestPrice),
|
||||
'offer_count'=> (int) ($offers['offerCount'] ?? count($offerList)),
|
||||
'offers' => $offerList,
|
||||
'crawled_at' => new UTCDateTime(time() * 1000),
|
||||
];
|
||||
}
|
||||
|
||||
private function applyToDb(string $slug, array $parsed): void
|
||||
{
|
||||
$now = time();
|
||||
|
||||
$existing = Mongo::collection('games')->findOne(['slug' => $slug]);
|
||||
$title = $parsed['title'] ?? ($existing['title'] ?? $slug);
|
||||
|
||||
// update the game doc with the gg.deals lowest price (upsert for new games)
|
||||
$set = [
|
||||
'last_gg_crawled' => $parsed['crawled_at'],
|
||||
'gg_best_price' => $parsed['best_price'],
|
||||
'title' => $title,
|
||||
'slug' => $slug,
|
||||
'source' => 'ggdeals',
|
||||
];
|
||||
if ($existing === null) {
|
||||
$set['genres'] = [];
|
||||
$set['platforms'] = ['pc'];
|
||||
$set['score'] = 0;
|
||||
$set['rating_pct'] = 0;
|
||||
$set['trending'] = 50;
|
||||
$set['cover'] = ['file' => '', 'from' => '#1b2838', 'to' => '#171d25'];
|
||||
$set['description'] = '';
|
||||
}
|
||||
if (! isset($existing['base_price']) || (float) $existing['base_price'] <= 0) {
|
||||
$set['base_price'] = $parsed['high_price'];
|
||||
}
|
||||
|
||||
// all-time low: keep the minimum of existing and gg.deals lowest
|
||||
$currentHl = $existing['all_time_low']['price'] ?? null;
|
||||
if ($currentHl === null || $parsed['best_price'] < (float) $currentHl) {
|
||||
$set['all_time_low'] = [
|
||||
'price' => $parsed['best_price'],
|
||||
'store' => 'gg.deals',
|
||||
'kind' => 'keyshop',
|
||||
'date' => new UTCDateTime($now * 1000),
|
||||
];
|
||||
}
|
||||
|
||||
Mongo::collection('games')->updateOne(['slug' => $slug], ['$set' => $set], ['upsert' => true]);
|
||||
|
||||
// store each offer as a deal (upsert by game+store)
|
||||
foreach ($parsed['offers'] as $o) {
|
||||
$storeCode = $this->storeCode($o['seller']);
|
||||
$expiry = $o['expiry'] !== null ? new UTCDateTime($o['expiry'] * 1000) : null;
|
||||
|
||||
Mongo::collection('deals')->updateOne(
|
||||
['game_slug' => $slug, 'store_code' => $storeCode, 'source' => 'ggdeals'],
|
||||
['$set' => [
|
||||
'game_slug' => $slug,
|
||||
'title' => $title,
|
||||
'store_code' => $storeCode,
|
||||
'kind' => 'keyshop',
|
||||
'price' => round($o['price'], 2),
|
||||
'regular' => round($parsed['high_price'], 2),
|
||||
'cut' => $parsed['high_price'] > 0 ? (int) round((1 - $o['price'] / $parsed['high_price']) * 100) : 0,
|
||||
'historical_low' => $o['price'] <= $parsed['best_price'] + 0.01,
|
||||
'drm' => 'GG.deals key',
|
||||
'regional' => false,
|
||||
'expiry' => $expiry,
|
||||
'url' => 'https://gg.deals/game/' . $slug . '/',
|
||||
'last_crawled' => new UTCDateTime($now * 1000),
|
||||
'source' => 'ggdeals',
|
||||
]],
|
||||
['upsert' => true]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Map a gg.deals seller name to a store code (best-effort).
|
||||
*/
|
||||
private function storeCode(string $seller): string
|
||||
{
|
||||
$map = [
|
||||
'steam' => 'steam', 'gog' => 'gog', 'gog.com' => 'gog',
|
||||
'epic' => 'epic', 'fanatical' => 'fanatical',
|
||||
'humble' => 'humble', 'green man gaming' => 'gmg', 'gmg' => 'gmg',
|
||||
'eneba' => 'eneba', 'kinguin' => 'kinguin', 'g2a' => 'g2a',
|
||||
'gamivo' => 'gamivo', 'cdkeys' => 'cdkeys', 'hrk game' => 'hrk',
|
||||
'instant gaming' => 'instantgaming', 'gamebillet' => 'gamebillet',
|
||||
'gamersgate' => 'gamersgate', 'gamesplanet' => 'gamesplanet',
|
||||
'k4g.com' => 'k4g', 'difmark' => 'difmark', 'driffle' => 'driffle',
|
||||
];
|
||||
$key = strtolower(trim($seller));
|
||||
foreach ($map as $k => $code) {
|
||||
if (str_contains($key, $k)) {
|
||||
return $code;
|
||||
}
|
||||
}
|
||||
|
||||
return 'gg_' . url_title($seller, '_', true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a command to the FlareSolverr API.
|
||||
*/
|
||||
private function fsRequest(array $payload): ?array
|
||||
{
|
||||
$ch = curl_init(self::FS_URL);
|
||||
curl_setopt_array($ch, [
|
||||
CURLOPT_RETURNTRANSFER => true,
|
||||
CURLOPT_TIMEOUT => 90,
|
||||
CURLOPT_POST => true,
|
||||
CURLOPT_POSTFIELDS => json_encode($payload),
|
||||
CURLOPT_HTTPHEADER => ['Content-Type: application/json'],
|
||||
CURLOPT_SSL_VERIFYPEER => false,
|
||||
]);
|
||||
$body = curl_exec($ch);
|
||||
$code = (int) curl_getinfo($ch, CURLINFO_RESPONSE_CODE);
|
||||
curl_close($ch);
|
||||
|
||||
if ($code !== 200 || $body === false) {
|
||||
CLI::error('FlareSolverr request failed: ' . $code);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
$json = json_decode($body, true);
|
||||
if (! is_array($json)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (($json['status'] ?? '') === 'error') {
|
||||
CLI::error('FlareSolverr error: ' . ($json['message'] ?? 'unknown'));
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
return $json;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user