feat: 8K Natural Lifestyle Photography Engine, 3-column value strip, smart cutout alpha matting, and full BN/EN localization
This commit is contained in:
khondokartowsif171
@@ -1,10 +1,22 @@
|
||||
const http = require('http');
|
||||
const https = require('https');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const os = require('os');
|
||||
const urlModule = require('url');
|
||||
const crypto = require('crypto');
|
||||
|
||||
const PORT = 8080;
|
||||
const DIRECTORY = __dirname;
|
||||
const DATA_DIR = path.join(DIRECTORY, 'data');
|
||||
const SYNCED_STORES_FILE = path.join(DATA_DIR, 'synced_stores.json');
|
||||
|
||||
if (!fs.existsSync(DATA_DIR)) {
|
||||
fs.mkdirSync(DATA_DIR, { recursive: true });
|
||||
}
|
||||
if (!fs.existsSync(SYNCED_STORES_FILE)) {
|
||||
fs.writeFileSync(SYNCED_STORES_FILE, JSON.stringify([], null, 2));
|
||||
}
|
||||
|
||||
const MIME_TYPES = {
|
||||
'.html': 'text/html; charset=UTF-8',
|
||||
@@ -30,10 +42,437 @@ function getLocalIp() {
|
||||
return '127.0.0.1';
|
||||
}
|
||||
|
||||
const server = http.createServer((req, res) => {
|
||||
// Utility: Fetch URL with Redirects & Browser User-Agent
|
||||
function fetchUrl(targetUrl, maxRedirects = 5) {
|
||||
return new Promise((resolve, reject) => {
|
||||
if (maxRedirects <= 0) return reject(new Error('Too many redirects'));
|
||||
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(targetUrl);
|
||||
} catch (e) {
|
||||
return reject(new Error(`Invalid URL: ${targetUrl}`));
|
||||
}
|
||||
|
||||
const client = parsed.protocol === 'https:' ? https : http;
|
||||
const req = client.get(targetUrl, {
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,application/json,image/webp,*/*;q=0.8',
|
||||
'Accept-Language': 'en-US,en;q=0.9,bn;q=0.8'
|
||||
},
|
||||
timeout: 10000
|
||||
}, (res) => {
|
||||
if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
|
||||
const nextUrl = new URL(res.headers.location, targetUrl).href;
|
||||
return fetchUrl(nextUrl, maxRedirects - 1).then(resolve).catch(reject);
|
||||
}
|
||||
|
||||
let data = '';
|
||||
res.setEncoding('utf8');
|
||||
res.on('data', chunk => { data += chunk; });
|
||||
res.on('end', () => {
|
||||
resolve({
|
||||
statusCode: res.statusCode,
|
||||
headers: res.headers,
|
||||
data: data,
|
||||
finalUrl: targetUrl
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
req.on('timeout', () => {
|
||||
req.destroy();
|
||||
reject(new Error('Request timed out'));
|
||||
});
|
||||
req.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
// Helper: Convert English digits to Bengali format for prices
|
||||
function toBengaliPrice(num) {
|
||||
const bnDigits = ['০', '১', '২', '৩', '৪', '৫', '৬', '৭', '৮', '৯'];
|
||||
const formattedNum = Math.round(num).toLocaleString('en-US');
|
||||
const bn = formattedNum.replace(/\d/g, d => bnDigits[d]);
|
||||
return `৳ ${bn}`;
|
||||
}
|
||||
|
||||
// Helper: Recursively search nested JSON / state / props for product items
|
||||
function findProductsInObject(obj, results, storeName, origin, category, storeId, depth = 0) {
|
||||
if (!obj || depth > 8 || results.length >= 120) return;
|
||||
|
||||
if (Array.isArray(obj)) {
|
||||
for (const item of obj) {
|
||||
if (typeof item === 'object' && item !== null) {
|
||||
const title = item.title || item.name || item.product_name || item.productTitle || '';
|
||||
const price = item.price || item.sale_price || item.regular_price || item.unit_price || (item.variants && item.variants[0] ? item.variants[0].price : 0);
|
||||
const image = item.image || item.thumbnail || item.featured_image || item.photo || (item.images && item.images[0] ? (item.images[0].src || item.images[0]) : '') || '';
|
||||
|
||||
if (title && typeof title === 'string' && title.length > 3 && (price || image)) {
|
||||
const numPrice = parseFloat(String(price).replace(/[^0-9.]/g, '')) || 1200;
|
||||
let imgUrl = typeof image === 'string' ? image : (image.url || image.src || '');
|
||||
if (imgUrl && !imgUrl.startsWith('http') && !imgUrl.startsWith('data:')) {
|
||||
try { imgUrl = new URL(imgUrl, origin).href; } catch(e) {}
|
||||
}
|
||||
|
||||
results.push({
|
||||
id: item.id ? String(item.id) : (item._id ? String(item._id) : crypto.randomUUID()),
|
||||
title: title.trim(),
|
||||
brand: item.brand?.name || item.brand || item.vendor || storeName,
|
||||
price: toBengaliPrice(numPrice),
|
||||
rawPrice: numPrice,
|
||||
regularPrice: toBengaliPrice(numPrice * 1.25),
|
||||
image: imgUrl,
|
||||
link: item.url || (item.handle ? `${origin}/products/${item.handle}` : origin),
|
||||
category: category,
|
||||
storeName: storeName,
|
||||
storeId: storeId
|
||||
});
|
||||
} else {
|
||||
findProductsInObject(item, results, storeName, origin, category, storeId, depth + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (typeof obj === 'object') {
|
||||
for (const key of Object.keys(obj)) {
|
||||
findProductsInObject(obj[key], results, storeName, origin, category, storeId, depth + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Multi-Strategy Real Store Scraper & Product Extractor
|
||||
async function scrapeStore(rawUrl, customStoreName = '', customCategory = 'general') {
|
||||
let targetUrl = rawUrl.trim();
|
||||
if (!targetUrl.startsWith('http://') && !targetUrl.startsWith('https://')) {
|
||||
targetUrl = 'https://' + targetUrl;
|
||||
}
|
||||
|
||||
const parsedUrl = new URL(targetUrl);
|
||||
const origin = parsedUrl.origin;
|
||||
const hostname = parsedUrl.hostname.replace(/^www\./, '');
|
||||
|
||||
// Clean default store name from domain
|
||||
const rawBrandFromDomain = hostname.split('.')[0];
|
||||
const autoStoreName = customStoreName || (rawBrandFromDomain.charAt(0).toUpperCase() + rawBrandFromDomain.slice(1));
|
||||
const storeId = 'store_' + hostname.replace(/[^a-zA-Z0-9]/g, '_').toLowerCase();
|
||||
|
||||
let products = [];
|
||||
let detectedTitle = autoStoreName;
|
||||
|
||||
// STRATEGY 1: Check Shopify products.json API (extremely fast and structured)
|
||||
try {
|
||||
const shopifyUrl = `${origin}/products.json?limit=250`;
|
||||
const shopifyRes = await fetchUrl(shopifyUrl);
|
||||
if (shopifyRes.statusCode === 200) {
|
||||
const shopifyData = JSON.parse(shopifyRes.data);
|
||||
if (shopifyData && Array.isArray(shopifyData.products) && shopifyData.products.length > 0) {
|
||||
shopifyData.products.forEach(p => {
|
||||
const variant = p.variants && p.variants[0] ? p.variants[0] : {};
|
||||
const priceVal = parseFloat(variant.price) || 1200;
|
||||
const regPriceVal = parseFloat(variant.compare_at_price) || (priceVal * 1.22);
|
||||
const imgUrl = (p.images && p.images[0] && p.images[0].src) ? p.images[0].src : ((p.image && p.image.src) ? p.image.src : '');
|
||||
|
||||
products.push({
|
||||
id: p.id ? String(p.id) : crypto.randomUUID(),
|
||||
title: p.title || 'Product',
|
||||
brand: p.vendor || autoStoreName,
|
||||
price: toBengaliPrice(priceVal),
|
||||
rawPrice: priceVal,
|
||||
regularPrice: toBengaliPrice(regPriceVal),
|
||||
image: imgUrl,
|
||||
link: `${origin}/products/${p.handle || p.id}`,
|
||||
category: customCategory,
|
||||
storeName: autoStoreName,
|
||||
storeId: storeId
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
// Non-shopify or shopify API restricted, continue to HTML scraper
|
||||
}
|
||||
|
||||
// STRATEGY 2: If products not found via Shopify API, fetch HTML and parse Schema.org JSON-LD & DOM
|
||||
if (products.length === 0) {
|
||||
const mainRes = await fetchUrl(targetUrl);
|
||||
if (mainRes.statusCode !== 200) {
|
||||
throw new Error(`Failed to fetch webpage (HTTP ${mainRes.statusCode})`);
|
||||
}
|
||||
|
||||
const html = mainRes.data;
|
||||
|
||||
// Detect Page Title
|
||||
const titleMatch = html.match(/<title[^>]*>([^<]+)<\/title>/i);
|
||||
if (titleMatch && titleMatch[1]) {
|
||||
const rawTitle = titleMatch[1].trim();
|
||||
detectedTitle = customStoreName || rawTitle.split(/[|\-–—]/)[0].trim() || autoStoreName;
|
||||
}
|
||||
|
||||
// 2A: Next.js __NEXT_DATA__ & Nuxt JSON State Extractor
|
||||
const nextDataMatch = html.match(/<script[^>]+id=["']__NEXT_DATA__["'][^>]*>([\s\S]*?)<\/script>/i) ||
|
||||
html.match(/<script[^>]*>window\.__INITIAL_STATE__\s*=\s*(\{[\s\S]*?\});<\/script>/i);
|
||||
if (nextDataMatch && nextDataMatch[1]) {
|
||||
try {
|
||||
const stateJson = JSON.parse(nextDataMatch[1]);
|
||||
findProductsInObject(stateJson, products, autoStoreName, origin, customCategory, storeId);
|
||||
} catch (e) {}
|
||||
}
|
||||
|
||||
// 2B: Schema.org JSON-LD Parsing
|
||||
const jsonLdMatches = html.matchAll(/<script[^>]+type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi);
|
||||
for (const match of jsonLdMatches) {
|
||||
try {
|
||||
const parsed = JSON.parse(match[1]);
|
||||
const items = Array.isArray(parsed) ? parsed : (parsed['@graph'] || [parsed]);
|
||||
|
||||
for (const item of items) {
|
||||
if (item['@type'] === 'Product') {
|
||||
const priceVal = item.offers ? (parseFloat(item.offers.price || item.offers.lowPrice) || 1500) : 1500;
|
||||
const regPriceVal = priceVal * 1.25;
|
||||
const imgUrl = Array.isArray(item.image) ? item.image[0] : (item.image?.url || item.image || '');
|
||||
|
||||
products.push({
|
||||
id: item.sku || crypto.randomUUID(),
|
||||
title: item.name || 'Product',
|
||||
brand: item.brand?.name || item.brand || autoStoreName,
|
||||
price: toBengaliPrice(priceVal),
|
||||
rawPrice: priceVal,
|
||||
regularPrice: toBengaliPrice(regPriceVal),
|
||||
image: typeof imgUrl === 'string' ? imgUrl : '',
|
||||
link: item.url || targetUrl,
|
||||
category: customCategory,
|
||||
storeName: autoStoreName,
|
||||
storeId: storeId
|
||||
});
|
||||
} else if (item['@type'] === 'ItemList' && Array.isArray(item.itemListElement)) {
|
||||
for (const elem of item.itemListElement) {
|
||||
const p = elem.item || elem;
|
||||
if (p.name) {
|
||||
const priceVal = p.offers ? (parseFloat(p.offers.price) || 1200) : 1200;
|
||||
products.push({
|
||||
id: crypto.randomUUID(),
|
||||
title: p.name,
|
||||
brand: p.brand?.name || autoStoreName,
|
||||
price: toBengaliPrice(priceVal),
|
||||
rawPrice: priceVal,
|
||||
regularPrice: toBengaliPrice(priceVal * 1.25),
|
||||
image: p.image || '',
|
||||
link: p.url || targetUrl,
|
||||
category: customCategory,
|
||||
storeName: autoStoreName,
|
||||
storeId: storeId
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
// Skip malformed JSON-LD block
|
||||
}
|
||||
}
|
||||
|
||||
// 2B: HTML Semantic Product Card Regex Parser (if JSON-LD had < 2 products)
|
||||
if (products.length < 2) {
|
||||
// Find product blocks
|
||||
const cardRegex = /<(div|article|li)[^>]*class=["'][^"']*(product|item|grid|card)[^"']*["'][^>]*>([\s\S]*?)<\/\1>/gi;
|
||||
let cardMatch;
|
||||
let count = 0;
|
||||
|
||||
while ((cardMatch = cardRegex.exec(html)) !== null && count < 60) {
|
||||
const block = cardMatch[3];
|
||||
|
||||
// Extract title
|
||||
const titleTag = block.match(/<(h2|h3|h4|h5|a)[^>]*class=["'][^"']*(title|name|heading)[^"']*["'][^>]*>([^<]+)<\/\1>/i) ||
|
||||
block.match(/<(h2|h3|h4|h5)[^>]*>([^<]+)<\/\1>/i) ||
|
||||
block.match(/<a[^>]*title=["']([^"']+)["']/i);
|
||||
const title = titleTag ? (titleTag[3] || titleTag[2] || titleTag[1]).trim() : '';
|
||||
|
||||
// Extract image
|
||||
const imgTag = block.match(/<img[^>]+(?:src|data-src|data-lazy-src)=["']([^"']+)["'][^>]*>/i);
|
||||
let imgUrl = imgTag ? imgTag[1] : '';
|
||||
if (imgUrl && !imgUrl.startsWith('http') && !imgUrl.startsWith('data:')) {
|
||||
imgUrl = new URL(imgUrl, origin).href;
|
||||
}
|
||||
|
||||
// Extract link
|
||||
const linkTag = block.match(/<a[^>]+href=["']([^"']+)["']/i);
|
||||
let linkUrl = linkTag ? linkTag[1] : targetUrl;
|
||||
if (linkUrl && !linkUrl.startsWith('http')) {
|
||||
linkUrl = new URL(linkUrl, origin).href;
|
||||
}
|
||||
|
||||
// Extract price
|
||||
const priceTag = block.match(/(?:৳|Tk\.?|BDT|\$)\s*([\d,]+(?:\.\d+)?)/i) ||
|
||||
block.match(/([\d,]+)\s*(?:৳|Tk\.?|BDT)/i);
|
||||
let priceVal = 1450;
|
||||
if (priceTag) {
|
||||
const rawNum = parseFloat((priceTag[1] || priceTag[2] || '').replace(/,/g, ''));
|
||||
if (rawNum && rawNum > 0) priceVal = rawNum;
|
||||
}
|
||||
|
||||
if (title && title.length > 2 && (imgUrl || block.includes('price'))) {
|
||||
count++;
|
||||
products.push({
|
||||
id: crypto.randomUUID(),
|
||||
title: title.replace(/&/g, '&').replace(/'/g, "'"),
|
||||
brand: autoStoreName,
|
||||
price: toBengaliPrice(priceVal),
|
||||
rawPrice: priceVal,
|
||||
regularPrice: toBengaliPrice(priceVal * 1.25),
|
||||
image: imgUrl,
|
||||
link: linkUrl,
|
||||
category: customCategory,
|
||||
storeName: autoStoreName,
|
||||
storeId: storeId
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Deduplicate products by title
|
||||
const seenTitles = new Set();
|
||||
products = products.filter(p => {
|
||||
const key = p.title.toLowerCase().trim();
|
||||
if (seenTitles.has(key)) return false;
|
||||
seenTitles.add(key);
|
||||
return true;
|
||||
});
|
||||
|
||||
const finalStore = {
|
||||
id: storeId,
|
||||
name: autoStoreName,
|
||||
detectedTitle: detectedTitle,
|
||||
domain: hostname,
|
||||
url: targetUrl,
|
||||
category: customCategory,
|
||||
syncedAt: new Date().toISOString(),
|
||||
count: products.length,
|
||||
products: products
|
||||
};
|
||||
|
||||
// Save to persistent storage
|
||||
let savedStores = [];
|
||||
try {
|
||||
if (fs.existsSync(SYNCED_STORES_FILE)) {
|
||||
savedStores = JSON.parse(fs.readFileSync(SYNCED_STORES_FILE, 'utf8'));
|
||||
}
|
||||
} catch (e) {
|
||||
savedStores = [];
|
||||
}
|
||||
|
||||
// Upsert store
|
||||
const existingIdx = savedStores.findIndex(s => s.id === storeId || s.domain === hostname);
|
||||
if (existingIdx >= 0) {
|
||||
savedStores[existingIdx] = finalStore;
|
||||
} else {
|
||||
savedStores.push(finalStore);
|
||||
}
|
||||
|
||||
fs.writeFileSync(SYNCED_STORES_FILE, JSON.stringify(savedStores, null, 2), 'utf8');
|
||||
|
||||
return finalStore;
|
||||
}
|
||||
|
||||
const server = http.createServer(async (req, res) => {
|
||||
let reqPath = req.url.split('?')[0];
|
||||
if (reqPath === '/' || reqPath === '') reqPath = '/index.html';
|
||||
|
||||
// API: Save Asset / Product Image
|
||||
if (req.method === 'POST' && reqPath === '/api/save-asset') {
|
||||
let body = '';
|
||||
req.on('data', chunk => { body += chunk; });
|
||||
req.on('end', () => {
|
||||
try {
|
||||
const { filename, base64Data } = JSON.parse(body);
|
||||
const buffer = Buffer.from(base64Data.replace(/^data:image\/\w+;base64,/, ''), 'base64');
|
||||
const savePath = path.join(DIRECTORY, 'assets', 'products', filename);
|
||||
fs.writeFileSync(savePath, buffer);
|
||||
res.writeHead(200, { 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*' });
|
||||
res.end(JSON.stringify({ success: true, savedTo: savePath }));
|
||||
} catch (e) {
|
||||
res.writeHead(500, { 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*' });
|
||||
res.end(JSON.stringify({ error: e.message }));
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// API: Universal Store URL Sync Endpoint
|
||||
if (req.method === 'POST' && reqPath === '/api/sync-store') {
|
||||
let body = '';
|
||||
req.on('data', chunk => { body += chunk; });
|
||||
req.on('end', async () => {
|
||||
try {
|
||||
const { url, storeName, category } = JSON.parse(body || '{}');
|
||||
if (!url) {
|
||||
res.writeHead(400, { 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*' });
|
||||
return res.end(JSON.stringify({ error: 'Store URL is required' }));
|
||||
}
|
||||
|
||||
const storeResult = await scrapeStore(url, storeName, category);
|
||||
res.writeHead(200, { 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*' });
|
||||
res.end(JSON.stringify({ success: true, store: storeResult }));
|
||||
} catch (e) {
|
||||
res.writeHead(500, { 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*' });
|
||||
res.end(JSON.stringify({ error: e.message }));
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// API: Get All Synced Stores
|
||||
if (req.method === 'GET' && reqPath === '/api/get-synced-stores') {
|
||||
try {
|
||||
let savedStores = [];
|
||||
if (fs.existsSync(SYNCED_STORES_FILE)) {
|
||||
savedStores = JSON.parse(fs.readFileSync(SYNCED_STORES_FILE, 'utf8'));
|
||||
}
|
||||
res.writeHead(200, { 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*' });
|
||||
res.end(JSON.stringify({ success: true, stores: savedStores }));
|
||||
} catch (e) {
|
||||
res.writeHead(500, { 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*' });
|
||||
res.end(JSON.stringify({ error: e.message }));
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// API: Delete Synced Store
|
||||
if (req.method === 'POST' && reqPath === '/api/delete-synced-store') {
|
||||
let body = '';
|
||||
req.on('data', chunk => { body += chunk; });
|
||||
req.on('end', () => {
|
||||
try {
|
||||
const { storeId } = JSON.parse(body || '{}');
|
||||
let savedStores = [];
|
||||
if (fs.existsSync(SYNCED_STORES_FILE)) {
|
||||
savedStores = JSON.parse(fs.readFileSync(SYNCED_STORES_FILE, 'utf8'));
|
||||
}
|
||||
savedStores = savedStores.filter(s => s.id !== storeId);
|
||||
fs.writeFileSync(SYNCED_STORES_FILE, JSON.stringify(savedStores, null, 2), 'utf8');
|
||||
res.writeHead(200, { 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*' });
|
||||
res.end(JSON.stringify({ success: true, stores: savedStores }));
|
||||
} catch (e) {
|
||||
res.writeHead(500, { 'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*' });
|
||||
res.end(JSON.stringify({ error: e.message }));
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// CORS Preflight
|
||||
if (req.method === 'OPTIONS') {
|
||||
res.writeHead(204, {
|
||||
'Access-Control-Allow-Origin': '*',
|
||||
'Access-Control-Allow-Methods': 'GET, POST, OPTIONS',
|
||||
'Access-Control-Allow-Headers': 'Content-Type'
|
||||
});
|
||||
return res.end();
|
||||
}
|
||||
|
||||
const filePath = path.join(DIRECTORY, reqPath);
|
||||
const ext = path.extname(filePath).toLowerCase();
|
||||
const contentType = MIME_TYPES[ext] || 'application/octet-stream';
|
||||
|
||||
Reference in New Issue
Block a user