GG.deals clone: CodeIgniter 4 + MongoDB game price tracker
- Full gg.deals-style UI: deal feed with filters, game pages with price history charts, rankings, vouchers, news, prepaids, subscriptions, bundles, franchises, wishlist, API docs - Real data crawlers: * gg:crawl (Steam Store API + GOG Catalog + Epic free games) * gg:catalog (252K game slugs from gg.deals sitemaps) * gg:ggdeals (per-game prices via FlareSolverr session, JSON-LD parsing) * gg:seed / gg:crawl-covers (demo seed + Steam CDN cover art) - 24/7 self-updating scheduler (scripts/scheduler_loop.php, lockfile-protected) - Tailwind v4 dark/light theme, Alpine.js, Chart.js
This commit is contained in:
deepseek
@@ -0,0 +1,661 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Commands;
|
||||
|
||||
use App\Libraries\Mongo;
|
||||
use CodeIgniter\CLI\BaseCommand;
|
||||
use CodeIgniter\CLI\CLI;
|
||||
use MongoDB\BSON\UTCDateTime;
|
||||
|
||||
/**
|
||||
* Crawls real game data & deals from public sources.
|
||||
*
|
||||
* php spark gg:crawl → all sources (steam, gog, epic)
|
||||
* php spark gg:crawl steam → Steam only
|
||||
* php spark gg:crawl gog → GOG only
|
||||
* php spark gg:crawl epic → Epic free games only
|
||||
*
|
||||
* Sources:
|
||||
* - Steam Store API (featured categories + app details, cc=us)
|
||||
* - GOG Catalog API (top rated products + prices)
|
||||
* - Epic Store free games promotions API
|
||||
*/
|
||||
class CrawlGames extends BaseCommand
|
||||
{
|
||||
protected $group = 'GGDeals';
|
||||
protected $name = 'gg:crawl';
|
||||
protected $description = 'Crawls real game & deal data from Steam, GOG and Epic.';
|
||||
protected $usage = 'gg:crawl [steam|gog|epic]';
|
||||
|
||||
private const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36';
|
||||
|
||||
private const STEAM_FEATURED = 'https://store.steampowered.com/api/featuredcategories/?cc=us&l=en';
|
||||
private const STEAM_APP_DETAILS = 'https://store.steampowered.com/api/appdetails/?cc=us&l=en&appids=%d&filters=basic,price_overview,genres,release_date,metacritic,developers,publishers,categories';
|
||||
private const STEAM_SEARCH = 'https://store.steampowered.com/search/results/?query&start=%d&count=100&dynamic_data=&sort_by=_ASC&cc=us&l=en&category1=998&infinite=1';
|
||||
private const GOG_CATALOG = 'https://catalog.gog.com/v1/catalog?limit=%d&order=desc:totalRating&productType=in:game&query=like:*';
|
||||
private const EPIC_FREE = 'https://store-site-backend-static.ak.epicgames.com/freeGamesPromotions?locale=en-US&country=US&allowCountries=US';
|
||||
|
||||
public function run(array $params): void
|
||||
{
|
||||
$target = strtolower($params[0] ?? 'all');
|
||||
|
||||
CLI::write('GG.deals crawler started — ' . date('Y-m-d H:i:s'), 'green');
|
||||
|
||||
if ($target === 'all' || $target === 'steam') {
|
||||
$this->crawlSteam();
|
||||
}
|
||||
if ($target === 'all' || $target === 'gog') {
|
||||
$this->crawlGog();
|
||||
}
|
||||
if ($target === 'all' || $target === 'epic') {
|
||||
$this->crawlEpic();
|
||||
}
|
||||
|
||||
if ($target === 'all' || $target === 'finalize' || in_array($target, ['steam', 'gog', 'epic'], true)) {
|
||||
$this->finalizeGames();
|
||||
$this->recordPriceHistory();
|
||||
}
|
||||
|
||||
CLI::write('Crawler finished — ' . date('Y-m-d H:i:s'), 'green');
|
||||
}
|
||||
|
||||
/**
|
||||
* Records today's cheapest price per game/store-kind into price_history,
|
||||
* so the charts grow real data over time.
|
||||
*/
|
||||
private function recordPriceHistory(): void
|
||||
{
|
||||
CLI::write('-- Recording price history --', 'cyan');
|
||||
|
||||
$todayStart = (new \DateTime('today'))->getTimestamp();
|
||||
$now = time();
|
||||
$inserted = 0;
|
||||
|
||||
$deals = Mongo::collection('deals')->find()->toArray();
|
||||
$byGame = [];
|
||||
|
||||
foreach ($deals as $d) {
|
||||
$byGame[$d['game_slug']][$d['kind']][] = (float) $d['price'];
|
||||
}
|
||||
|
||||
foreach ($byGame as $slug => $kinds) {
|
||||
foreach (['official', 'keyshop'] as $kind) {
|
||||
if (empty($kinds[$kind])) {
|
||||
continue;
|
||||
}
|
||||
$min = min($kinds[$kind]);
|
||||
|
||||
// only insert one point per game+kind+day
|
||||
$existing = Mongo::collection('price_history')->findOne([
|
||||
'game_slug' => $slug,
|
||||
'kind' => $kind,
|
||||
'ts' => ['$gte' => new UTCDateTime($todayStart * 1000)],
|
||||
]);
|
||||
if ($existing !== null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
Mongo::collection('price_history')->insertOne([
|
||||
'game_slug' => $slug,
|
||||
'store_code' => 'aggregate',
|
||||
'kind' => $kind,
|
||||
'ts' => new UTCDateTime($now * 1000),
|
||||
'price' => round($min, 2),
|
||||
]);
|
||||
$inserted++;
|
||||
}
|
||||
}
|
||||
|
||||
CLI::write("Recorded {$inserted} price history points.", 'green');
|
||||
}
|
||||
|
||||
/**
|
||||
* Recomputes per-game aggregate fields (best prices, active deal counts,
|
||||
* all-time low) from the deals collection — keeps views working even when
|
||||
* a game has no official/keyshop store deals.
|
||||
*/
|
||||
private function finalizeGames(): void
|
||||
{
|
||||
CLI::write('-- Finalizing game aggregates --', 'cyan');
|
||||
|
||||
$deals = Mongo::collection('deals')->find()->toArray();
|
||||
$byGame = [];
|
||||
|
||||
foreach ($deals as $d) {
|
||||
$slug = $d['game_slug'];
|
||||
$byGame[$slug][] = $d;
|
||||
}
|
||||
|
||||
$updated = 0;
|
||||
$now = time();
|
||||
|
||||
foreach ($byGame as $slug => $gameDeals) {
|
||||
$official = array_values(array_filter($gameDeals, static fn ($d) => $d['kind'] === 'official'));
|
||||
$keyshop = array_values(array_filter($gameDeals, static fn ($d) => $d['kind'] === 'keyshop'));
|
||||
|
||||
usort($official, static fn ($a, $b) => $a['price'] <=> $b['price']);
|
||||
usort($keyshop, static fn ($a, $b) => $a['price'] <=> $b['price']);
|
||||
usort($gameDeals, static fn ($a, $b) => $a['price'] <=> $b['price']);
|
||||
|
||||
$bestOfficial = $official[0] ?? null;
|
||||
$bestKeyshop = $keyshop[0] ?? null;
|
||||
$cheapest = $gameDeals[0] ?? null;
|
||||
|
||||
$existing = Mongo::collection('games')->findOne(['slug' => $slug]);
|
||||
if ($existing === null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// all-time low: keep the historical minimum, seed from current cheapest if none
|
||||
$hl = $existing['all_time_low'] ?? null;
|
||||
if ($cheapest !== null && ($hl === null || (float) $cheapest['price'] < (float) $hl['price'])) {
|
||||
$hl = [
|
||||
'price' => (float) $cheapest['price'],
|
||||
'store' => $cheapest['store_code'],
|
||||
'kind' => $cheapest['kind'],
|
||||
'date' => new UTCDateTime($now * 1000),
|
||||
];
|
||||
}
|
||||
|
||||
$set = [
|
||||
'best_official' => [
|
||||
'price' => $bestOfficial ? (float) $bestOfficial['price'] : 0.0,
|
||||
'store' => $bestOfficial ? $bestOfficial['store_code'] : '',
|
||||
'cut' => $bestOfficial ? (int) $bestOfficial['cut'] : 0,
|
||||
],
|
||||
'best_keyshop' => [
|
||||
'price' => $bestKeyshop ? (float) $bestKeyshop['price'] : 0.0,
|
||||
'store' => $bestKeyshop ? $bestKeyshop['store_code'] : '',
|
||||
'cut' => $bestKeyshop ? (int) $bestKeyshop['cut'] : 0,
|
||||
],
|
||||
'active_deals' => count($gameDeals),
|
||||
'all_time_low' => $hl,
|
||||
'trending' => $existing['trending'] ?? mt_rand(30, 90),
|
||||
];
|
||||
|
||||
Mongo::collection('games')->updateOne(['slug' => $slug], ['$set' => $set]);
|
||||
$updated++;
|
||||
}
|
||||
|
||||
CLI::write("Finalized {$updated} games with aggregates.", 'green');
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ Steam
|
||||
|
||||
private function crawlSteam(): void
|
||||
{
|
||||
CLI::write('-- Steam --', 'cyan');
|
||||
|
||||
$data = $this->fetchJson(self::STEAM_FEATURED);
|
||||
if ($data === null) {
|
||||
CLI::error('Steam featured fetch failed');
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
$appIds = [];
|
||||
$featured = [];
|
||||
|
||||
foreach (['specials', 'top_sellers', 'new_releases', 'coming_soon'] as $cat) {
|
||||
foreach ($data[$cat]['items'] ?? [] as $item) {
|
||||
$appIds[$item['id']] = true;
|
||||
$featured[$item['id']] = [
|
||||
'category' => $cat,
|
||||
'discount_percent' => (int) ($item['discount_percent'] ?? 0),
|
||||
'final_price' => (int) ($item['final_price'] ?? 0),
|
||||
'original_price' => (int) ($item['original_price'] ?? 0),
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
// also add our existing appids so we refresh their details
|
||||
foreach (\App\Commands\CrawlCovers::APP_IDS as $slug => $appid) {
|
||||
$appIds[$appid] = true;
|
||||
}
|
||||
|
||||
// crawl Steam search catalog (games only) — several pages of 50
|
||||
$searchPages = (int) env('GG_CRAWL_STEAM_PAGES', '4');
|
||||
foreach ($this->steamSearchAppIds($searchPages) as $appid) {
|
||||
$appIds[$appid] = true;
|
||||
}
|
||||
|
||||
CLI::write('Found ' . count($appIds) . ' Steam apps to fetch details for.', 'yellow');
|
||||
|
||||
$gamesInserted = 0;
|
||||
$gamesUpdated = 0;
|
||||
$dealsInserted = 0;
|
||||
$dealsUpdated = 0;
|
||||
|
||||
foreach (array_keys($appIds) as $i => $appid) {
|
||||
$details = $this->fetchJson(sprintf(self::STEAM_APP_DETAILS, (int) $appid));
|
||||
if ($details === null || empty($details[(string) $appid]['success'])) {
|
||||
continue;
|
||||
}
|
||||
$g = $details[(string) $appid]['data'] ?? null;
|
||||
if ($g === null || empty($g['name'])) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$slug = url_title($g['name'], '-', true);
|
||||
if ($slug === '') {
|
||||
continue;
|
||||
}
|
||||
|
||||
$priceOverview = $g['price_overview'] ?? [];
|
||||
$priceUsd = isset($priceOverview['final']) ? (float) $priceOverview['final'] / 100 : null;
|
||||
$baseUsd = isset($priceOverview['initial']) ? (float) $priceOverview['initial'] / 100 : null;
|
||||
$cut = (int) ($priceOverview['discount_percent'] ?? 0);
|
||||
|
||||
$genres = array_map(static fn ($x) => $x['description'], $g['genres'] ?? []);
|
||||
$platforms = ['steam'];
|
||||
foreach ($g['platforms'] ?? [] as $plat => $supported) {
|
||||
if ($supported) {
|
||||
$platforms[] = $plat;
|
||||
}
|
||||
}
|
||||
|
||||
$now = time();
|
||||
$cover = $this->steamCoverUrl((int) $appid);
|
||||
|
||||
$existing = Mongo::collection('games')->findOne(['slug' => $slug]);
|
||||
$oldBase = $existing ? (float) $existing['base_price'] : ($baseUsd ?? 0);
|
||||
|
||||
$doc = [
|
||||
'slug' => $slug,
|
||||
'title' => $g['name'],
|
||||
'steam_appid' => (int) $appid,
|
||||
'base_price' => $baseUsd ?? $oldBase,
|
||||
'genres' => array_values(array_unique(array_filter($genres))),
|
||||
'platforms' => array_values(array_unique($platforms)),
|
||||
'score' => (int) ($g['metacritic']['score'] ?? 0),
|
||||
'rating_pct' => (int) ($g['metacritic']['score'] ?? 0),
|
||||
'release_date' => $this->parseDate($g['release_date']['date'] ?? null),
|
||||
'developer' => ($g['developers'][0] ?? null) ?: 'Unknown',
|
||||
'publisher' => ($g['publishers'][0] ?? null) ?: 'Unknown',
|
||||
'cover' => ['file' => $cover, 'from' => '#1b2838', 'to' => '#171d25'],
|
||||
'description' => $g['short_description'] ?? '',
|
||||
'age_rating' => 'Steam',
|
||||
'last_crawled' => new UTCDateTime($now * 1000),
|
||||
'source' => 'steam',
|
||||
];
|
||||
|
||||
// preserve existing all_time_low / best prices if present
|
||||
if ($existing) {
|
||||
foreach (['all_time_low', 'best_official', 'best_keyshop', 'active_deals', 'trending'] as $f) {
|
||||
if (isset($existing[$f])) {
|
||||
$doc[$f] = $existing[$f];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$result = Mongo::collection('games')->updateOne(
|
||||
['slug' => $slug],
|
||||
['$set' => $doc],
|
||||
['upsert' => true]
|
||||
);
|
||||
if ($result->getUpsertedCount() > 0) {
|
||||
$gamesInserted++;
|
||||
} else {
|
||||
$gamesUpdated++;
|
||||
}
|
||||
|
||||
// ---- deal (official store = Steam)
|
||||
if ($priceUsd !== null && $baseUsd !== null && $baseUsd > 0) {
|
||||
$hlPrice = $existing['all_time_low']['price'] ?? null;
|
||||
$dealDoc = [
|
||||
'game_slug' => $slug,
|
||||
'title' => $g['name'],
|
||||
'store_code' => 'steam',
|
||||
'kind' => 'official',
|
||||
'price' => round($priceUsd, 2),
|
||||
'regular' => round($baseUsd, 2),
|
||||
'cut' => max(0, min(95, $cut)),
|
||||
'historical_low' => $hlPrice !== null && $priceUsd <= $hlPrice + 0.01,
|
||||
'drm' => 'Steam key',
|
||||
'regional' => false,
|
||||
'expiry' => null,
|
||||
'url' => 'https://store.steampowered.com/app/' . (int) $appid . '/',
|
||||
'last_crawled' => new UTCDateTime($now * 1000),
|
||||
'source' => 'steam',
|
||||
];
|
||||
$dResult = Mongo::collection('deals')->updateOne(
|
||||
['game_slug' => $slug, 'store_code' => 'steam'],
|
||||
['$set' => $dealDoc],
|
||||
['upsert' => true]
|
||||
);
|
||||
if ($dResult->getUpsertedCount() > 0) {
|
||||
$dealsInserted++;
|
||||
} else {
|
||||
$dealsUpdated++;
|
||||
}
|
||||
}
|
||||
|
||||
if ($i % 25 === 0) {
|
||||
CLI::write(" ... {$i}/" . count($appIds));
|
||||
}
|
||||
usleep(200000); // rate limit ~5 req/s
|
||||
}
|
||||
|
||||
CLI::write("Steam done: {$gamesInserted} games inserted, {$gamesUpdated} updated, {$dealsInserted} deals inserted, {$dealsUpdated} deals updated.", 'green');
|
||||
}
|
||||
|
||||
private function steamCoverUrl(int $appid): string
|
||||
{
|
||||
return "https://shared.akamai.steamstatic.com/store_item_assets/steam/apps/{$appid}/header.jpg";
|
||||
}
|
||||
|
||||
/**
|
||||
* Collects app IDs from the Steam store search pages (category1=998 → games).
|
||||
*/
|
||||
private function steamSearchAppIds(int $pages): array
|
||||
{
|
||||
$ids = [];
|
||||
for ($page = 0; $page < $pages; $page++) {
|
||||
$url = sprintf(self::STEAM_SEARCH, $page * 100);
|
||||
$ch = curl_init($url);
|
||||
curl_setopt_array($ch, [
|
||||
CURLOPT_RETURNTRANSFER => true,
|
||||
CURLOPT_TIMEOUT => 30,
|
||||
CURLOPT_FOLLOWLOCATION => true,
|
||||
CURLOPT_USERAGENT => self::UA,
|
||||
CURLOPT_SSL_VERIFYPEER => true,
|
||||
]);
|
||||
$body = curl_exec($ch);
|
||||
$code = (int) curl_getinfo($ch, CURLINFO_RESPONSE_CODE);
|
||||
curl_close($ch);
|
||||
|
||||
if ($code !== 200 || $body === false) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// results_html is JSON-escaped: data-ds-appid=\"123456\"
|
||||
if (preg_match_all('/data-ds-appid=\\\\"(\d{3,9})\\\\"/', $body, $m)
|
||||
|| preg_match_all('/data-ds-appid="(\d{3,9})"/', $body, $m)) {
|
||||
foreach ($m[1] as $appid) {
|
||||
$ids[(int) $appid] = true;
|
||||
}
|
||||
}
|
||||
|
||||
CLI::write(" search page {$page}: " . count($ids) . ' unique ids so far');
|
||||
usleep(300000);
|
||||
}
|
||||
|
||||
return array_keys($ids);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ GOG
|
||||
|
||||
private function crawlGog(): void
|
||||
{
|
||||
CLI::write('-- GOG --', 'cyan');
|
||||
|
||||
$limit = 48;
|
||||
$pages = (int) env('GG_CRAWL_GOG_PAGES', '6');
|
||||
$inserted = 0;
|
||||
$updated = 0;
|
||||
$dealsInserted = 0;
|
||||
$now = time();
|
||||
|
||||
for ($page = 0; $page < $pages; $page++) {
|
||||
$url = sprintf(self::GOG_CATALOG, $limit) . '&page=' . $page . '&pageSize=' . $limit;
|
||||
$data = $this->fetchJson($url);
|
||||
if ($data === null || empty($data['products'])) {
|
||||
CLI::write(" GOG page {$page}: no products, stopping.");
|
||||
break;
|
||||
}
|
||||
CLI::write(" GOG page {$page}: " . count($data['products']) . ' products');
|
||||
|
||||
foreach ($data['products'] as $p) {
|
||||
$title = $p['title'] ?? null;
|
||||
if ($title === null) {
|
||||
continue;
|
||||
}
|
||||
$slug = url_title($title, '-', true);
|
||||
if ($slug === '') {
|
||||
continue;
|
||||
}
|
||||
|
||||
$price = $p['price'] ?? [];
|
||||
$baseUsd = isset($price['baseMoney']['amount']) ? (float) $price['baseMoney']['amount'] : null;
|
||||
$finalUsd = isset($price['finalMoney']['amount']) ? (float) $price['finalMoney']['amount'] : null;
|
||||
$cut = (int) rtrim((string) ($price['discount'] ?? '0%'), '%');
|
||||
|
||||
$genres = array_map(static fn ($x) => $x['name'], $p['genres'] ?? []);
|
||||
|
||||
$existing = Mongo::collection('games')->findOne(['slug' => $slug]);
|
||||
|
||||
$doc = [
|
||||
'slug' => $slug,
|
||||
'title' => $title,
|
||||
'base_price' => $baseUsd ?? 0,
|
||||
'genres' => array_values(array_unique(array_filter($genres))),
|
||||
'platforms' => ['gog'],
|
||||
'score' => (int) ($p['reviewsRating'] ?? 0),
|
||||
'rating_pct' => (int) ($p['reviewsRating'] ?? 0),
|
||||
'release_date' => $this->parseDate($p['releaseDate'] ?? null, 'Y.m.d'),
|
||||
'developer' => ($p['developers'][0] ?? null) ?: 'Unknown',
|
||||
'publisher' => ($p['publishers'][0] ?? null) ?: 'Unknown',
|
||||
'cover' => ['file' => $p['coverHorizontal'] ?? '', 'from' => '#862a52', 'to' => '#4b1530'],
|
||||
'description' => $p['description'] ?? '',
|
||||
'last_crawled' => new UTCDateTime($now * 1000),
|
||||
'source' => 'gog',
|
||||
];
|
||||
if ($existing) {
|
||||
foreach (['all_time_low', 'best_official', 'best_keyshop', 'active_deals', 'trending'] as $f) {
|
||||
if (isset($existing[$f])) {
|
||||
$doc[$f] = $existing[$f];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$r = Mongo::collection('games')->updateOne(['slug' => $slug], ['$set' => $doc], ['upsert' => true]);
|
||||
if ($r->getUpsertedCount() > 0) {
|
||||
$inserted++;
|
||||
} else {
|
||||
$updated++;
|
||||
}
|
||||
|
||||
if ($finalUsd !== null && $baseUsd !== null && $baseUsd > 0) {
|
||||
$hlPrice = $existing['all_time_low']['price'] ?? null;
|
||||
Mongo::collection('deals')->updateOne(
|
||||
['game_slug' => $slug, 'store_code' => 'gog'],
|
||||
['$set' => [
|
||||
'game_slug' => $slug,
|
||||
'title' => $title,
|
||||
'store_code' => 'gog',
|
||||
'kind' => 'official',
|
||||
'price' => round($finalUsd, 2),
|
||||
'regular' => round($baseUsd, 2),
|
||||
'cut' => max(0, min(95, $cut)),
|
||||
'historical_low' => $hlPrice !== null && $finalUsd <= $hlPrice + 0.01,
|
||||
'drm' => 'DRM-free',
|
||||
'regional' => false,
|
||||
'expiry' => null,
|
||||
'url' => $p['storeLink'] ?? 'https://www.gog.com/',
|
||||
'last_crawled' => new UTCDateTime($now * 1000),
|
||||
'source' => 'gog',
|
||||
]],
|
||||
['upsert' => true]
|
||||
);
|
||||
$dealsInserted++;
|
||||
}
|
||||
|
||||
usleep(100000);
|
||||
}
|
||||
}
|
||||
|
||||
CLI::write("GOG done: {$inserted} inserted, {$updated} updated, {$dealsInserted} deals.", 'green');
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ Epic
|
||||
|
||||
private function crawlEpic(): void
|
||||
{
|
||||
CLI::write('-- Epic free games --', 'cyan');
|
||||
|
||||
$data = $this->fetchJson(self::EPIC_FREE);
|
||||
if ($data === null) {
|
||||
CLI::error('Epic free fetch failed');
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
$elements = $data['data']['Catalog']['searchStore']['elements'] ?? [];
|
||||
$now = time();
|
||||
$inserted = 0;
|
||||
$dealsInserted = 0;
|
||||
|
||||
foreach ($elements as $e) {
|
||||
$title = $e['title'] ?? null;
|
||||
if ($title === null) {
|
||||
continue;
|
||||
}
|
||||
$slug = url_title($title, '-', true);
|
||||
if ($slug === '') {
|
||||
continue;
|
||||
}
|
||||
|
||||
$price = $e['price']['totalPrice']['fmtPrice'] ?? [];
|
||||
$orig = (float) str_replace(['$', ','], '', (string) ($price['originalPrice'] ?? '0'));
|
||||
$final = (float) str_replace(['$', ','], '', (string) ($price['discountPrice'] ?? '0'));
|
||||
|
||||
$promos = $e['promotions'] ?? [];
|
||||
$active = $promos['promotionalOffers'][0]['promotionalOffers'] ?? [];
|
||||
$upcoming = $promos['upcomingPromotionalOffers'][0]['promotionalOffers'] ?? [];
|
||||
$isFree = $final <= 0 || $active !== [] || $upcoming !== [];
|
||||
|
||||
$existing = Mongo::collection('games')->findOne(['slug' => $slug]);
|
||||
|
||||
$doc = [
|
||||
'slug' => $slug,
|
||||
'title' => $title,
|
||||
'base_price' => $orig,
|
||||
'genres' => [],
|
||||
'platforms' => ['epic'],
|
||||
'score' => 0,
|
||||
'rating_pct' => 0,
|
||||
'release_date' => new UTCDateTime($now * 1000),
|
||||
'developer' => ($e['developer'] ?? 'Unknown') ?: 'Unknown',
|
||||
'publisher' => ($e['publisher'] ?? 'Unknown') ?: 'Unknown',
|
||||
'cover' => ['file' => $e['keyImages'][0]['url'] ?? '', 'from' => '#2b2b2b', 'to' => '#1a1a1a'],
|
||||
'description' => $e['description'] ?? '',
|
||||
'last_crawled' => new UTCDateTime($now * 1000),
|
||||
'source' => 'epic',
|
||||
];
|
||||
if ($existing) {
|
||||
foreach (['all_time_low', 'best_official', 'best_keyshop', 'active_deals', 'trending'] as $f) {
|
||||
if (isset($existing[$f])) {
|
||||
$doc[$f] = $existing[$f];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$r = Mongo::collection('games')->updateOne(['slug' => $slug], ['$set' => $doc], ['upsert' => true]);
|
||||
if ($r->getUpsertedCount() > 0) {
|
||||
$inserted++;
|
||||
}
|
||||
|
||||
if ($final >= 0 && $orig > 0) {
|
||||
$expiry = null;
|
||||
foreach (array_merge($active, $upcoming) as $o) {
|
||||
if (! empty($o['endDate'])) {
|
||||
$expiry = new UTCDateTime(strtotime($o['endDate']) * 1000);
|
||||
break;
|
||||
}
|
||||
}
|
||||
Mongo::collection('deals')->updateOne(
|
||||
['game_slug' => $slug, 'store_code' => 'epic'],
|
||||
['$set' => [
|
||||
'game_slug' => $slug,
|
||||
'title' => $title,
|
||||
'store_code' => 'epic',
|
||||
'kind' => 'official',
|
||||
'price' => round($final, 2),
|
||||
'regular' => round($orig, 2),
|
||||
'cut' => $isFree ? 100 : max(0, min(95, (int) round((1 - $final / $orig) * 100))),
|
||||
'historical_low' => $isFree,
|
||||
'drm' => 'Epic key',
|
||||
'regional' => false,
|
||||
'expiry' => $expiry,
|
||||
'url' => 'https://store.epicgames.com/p/' . ($e['productSlug'] ?? $slug),
|
||||
'last_crawled' => new UTCDateTime($now * 1000),
|
||||
'source' => 'epic',
|
||||
]],
|
||||
['upsert' => true]
|
||||
);
|
||||
$dealsInserted++;
|
||||
|
||||
// free game → news freebie post (only when free and not already posted)
|
||||
if ($isFree && $existing === null) {
|
||||
$newsSlug = 'freebie-' . $slug;
|
||||
if (Mongo::collection('news')->findOne(['slug' => $newsSlug]) === null) {
|
||||
Mongo::collection('news')->insertOne([
|
||||
'title' => $title . ' is FREE to keep on Epic Games Store',
|
||||
'slug' => $newsSlug,
|
||||
'category' => 'freebies',
|
||||
'body' => "You can claim {$title} for free on the Epic Games Store right now — it's yours to keep forever once claimed. Head over to the store, sign in and hit the claim button before the offer expires.",
|
||||
'game_slug' => $slug,
|
||||
'date' => new UTCDateTime($now * 1000),
|
||||
]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
usleep(150000);
|
||||
}
|
||||
|
||||
CLI::write("Epic done: {$inserted} games inserted, {$dealsInserted} deals.", 'green');
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ helpers
|
||||
|
||||
private function fetchJson(string $url): ?array
|
||||
{
|
||||
$ch = curl_init($url);
|
||||
curl_setopt_array($ch, [
|
||||
CURLOPT_RETURNTRANSFER => true,
|
||||
CURLOPT_TIMEOUT => 30,
|
||||
CURLOPT_FOLLOWLOCATION => true,
|
||||
CURLOPT_USERAGENT => self::UA,
|
||||
CURLOPT_SSL_VERIFYPEER => true,
|
||||
CURLOPT_HTTPHEADER => ['Accept: application/json'],
|
||||
]);
|
||||
$body = curl_exec($ch);
|
||||
$code = (int) curl_getinfo($ch, CURLINFO_RESPONSE_CODE);
|
||||
curl_close($ch);
|
||||
|
||||
if ($code !== 200 || $body === false) {
|
||||
CLI::error("Fetch failed ({$code}): {$url}");
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
$json = json_decode($body, true);
|
||||
if (! is_array($json)) {
|
||||
CLI::error('Invalid JSON: ' . substr($body, 0, 100));
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
return $json;
|
||||
}
|
||||
|
||||
private function parseDate(?string $date, string $format = 'd M, Y'): ?UTCDateTime
|
||||
{
|
||||
if ($date === null || $date === '' || str_contains($date, 'Coming soon')) {
|
||||
return null;
|
||||
}
|
||||
$ts = \DateTime::createFromFormat($format, $date);
|
||||
if ($ts === false) {
|
||||
try {
|
||||
$ts = new \DateTime($date);
|
||||
} catch (\Exception) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
return new UTCDateTime($ts->getTimestamp() * 1000);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user