csv import

This commit is contained in:
2024-12-20 09:49:54 -05:00
parent d03bcf5746
commit 0eb54c6024
30 changed files with 10068 additions and 300 deletions

101
package-lock.json generated
View File

@@ -29,25 +29,31 @@
"lucide-react": "^0.460.0",
"next": "15.1.0",
"next-themes": "^0.4.3",
"path": "^0.12.7",
"pg": "^8.13.1",
"react": "18.2.0",
"react-dom": "18.2.0",
"react-icons": "^5.3.0",
"tailwind-merge": "^2.5.4",
"tailwindcss-animate": "^1.0.7"
"tailwindcss-animate": "^1.0.7",
"uuid": "^11.0.3"
},
"devDependencies": {
"@auth/drizzle-adapter": "^1.7.4",
"@types/bun": "^1.1.13",
"@types/node": "^20.17.6",
"@types/node": "^20.17.10",
"@types/pg": "^8.11.10",
"@types/react": "^18",
"@types/react-dom": "^18",
"@types/uuid": "^10.0.0",
"autoprefixer": "^10.4.20",
"csv-parse": "^5.6.0",
"csv-parser": "^3.0.0",
"drizzle-kit": "^0.28.1",
"drizzle-orm": "^0.38.2",
"eslint": "^8",
"eslint-config-next": "15.0.3",
"fs": "^0.0.1-security",
"next-auth": "^5.0.0-beta.25",
"postcss": "^8",
"tailwindcss": "^3.4.15",
@@ -2949,10 +2955,11 @@
"dev": true
},
"node_modules/@types/node": {
"version": "20.17.6",
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.17.6.tgz",
"integrity": "sha512-VEI7OdvK2wP7XHnsuXbAJnEpEkF6NjSN45QJlL4VGqZSXsnicpesdTWsg9RISeSdYd3yeRj/y3k5KGjUXYnFwQ==",
"version": "20.17.10",
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.17.10.tgz",
"integrity": "sha512-/jrvh5h6NXhEauFFexRin69nA0uHJ5gwk4iDivp/DeoEua3uwCUto6PC86IpRITBOs4+6i2I56K5x5b6WYGXHA==",
"dev": true,
"license": "MIT",
"dependencies": {
"undici-types": "~6.19.2"
}
@@ -3062,6 +3069,13 @@
"@types/react": "*"
}
},
"node_modules/@types/uuid": {
"version": "10.0.0",
"resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-10.0.0.tgz",
"integrity": "sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==",
"dev": true,
"license": "MIT"
},
"node_modules/@types/ws": {
"version": "8.5.13",
"resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.13.tgz",
@@ -4032,6 +4046,29 @@
"resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.3.tgz",
"integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw=="
},
"node_modules/csv-parse": {
"version": "5.6.0",
"resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.6.0.tgz",
"integrity": "sha512-l3nz3euub2QMg5ouu5U09Ew9Wf6/wQ8I++ch1loQ0ljmzhmfZYrH9fflS22i/PQEvsPvxCwxgz5q7UB8K1JO4Q==",
"dev": true,
"license": "MIT"
},
"node_modules/csv-parser": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/csv-parser/-/csv-parser-3.0.0.tgz",
"integrity": "sha512-s6OYSXAK3IdKqYO33y09jhypG/bSDHPuyCme/IdEHfWpLf/jKcpitVFyOC6UemgGk8v7Q5u2XE0vvwmanxhGlQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"minimist": "^1.2.0"
},
"bin": {
"csv-parser": "bin/csv-parser"
},
"engines": {
"node": ">= 10"
}
},
"node_modules/damerau-levenshtein": {
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/damerau-levenshtein/-/damerau-levenshtein-1.0.8.tgz",
@@ -5654,6 +5691,13 @@
}
}
},
"node_modules/fs": {
"version": "0.0.1-security",
"resolved": "https://registry.npmjs.org/fs/-/fs-0.0.1-security.tgz",
"integrity": "sha512-3XY9e1pP0CVEUCdj5BmfIZxRBTSDycnbqhIOGec9QYtmVH2fbLpj86CFWkrNOkt/Fvty4KZG5lTglL9j/gJ87w==",
"dev": true,
"license": "ISC"
},
"node_modules/fs.realpath": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
@@ -7154,6 +7198,16 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/path": {
"version": "0.12.7",
"resolved": "https://registry.npmjs.org/path/-/path-0.12.7.tgz",
"integrity": "sha512-aXXC6s+1w7otVF9UletFkFcDsJeO7lSZBPUQhtb5O0xJe8LtYhj/GxldoL09bBj9+ZmE2hNoHqQSFMN5fikh4Q==",
"license": "MIT",
"dependencies": {
"process": "^0.11.1",
"util": "^0.10.3"
}
},
"node_modules/path-exists": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz",
@@ -7588,6 +7642,15 @@
"fsevents": "2.3.3"
}
},
"node_modules/process": {
"version": "0.11.10",
"resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz",
"integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==",
"license": "MIT",
"engines": {
"node": ">= 0.6.0"
}
},
"node_modules/prop-types": {
"version": "15.8.1",
"resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz",
@@ -8735,11 +8798,39 @@
"punycode": "^2.1.0"
}
},
"node_modules/util": {
"version": "0.10.4",
"resolved": "https://registry.npmjs.org/util/-/util-0.10.4.tgz",
"integrity": "sha512-0Pm9hTQ3se5ll1XihRic3FDIku70C+iHUdT/W926rSgHV5QgXsYbKZN8MSC3tJtSkhuROzvsQjAaFENRXr+19A==",
"license": "MIT",
"dependencies": {
"inherits": "2.0.3"
}
},
"node_modules/util-deprecate": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
"integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="
},
"node_modules/util/node_modules/inherits": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
"integrity": "sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==",
"license": "ISC"
},
"node_modules/uuid": {
"version": "11.0.3",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-11.0.3.tgz",
"integrity": "sha512-d0z310fCWv5dJwnX1Y/MncBAqGMKEzlBb1AOf7z9K8ALnd0utBX/msg/fA0+sbyN1ihbMsLhrBlnl1ak7Wa0rg==",
"funding": [
"https://github.com/sponsors/broofa",
"https://github.com/sponsors/ctavan"
],
"license": "MIT",
"bin": {
"uuid": "dist/esm/bin/uuid"
}
},
"node_modules/which": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",

View File

@@ -30,25 +30,31 @@
"lucide-react": "^0.460.0",
"next": "15.1.0",
"next-themes": "^0.4.3",
"path": "^0.12.7",
"pg": "^8.13.1",
"react": "18.2.0",
"react-dom": "18.2.0",
"react-icons": "^5.3.0",
"tailwind-merge": "^2.5.4",
"tailwindcss-animate": "^1.0.7"
"tailwindcss-animate": "^1.0.7",
"uuid": "^11.0.3"
},
"devDependencies": {
"@auth/drizzle-adapter": "^1.7.4",
"@types/bun": "^1.1.13",
"@types/node": "^20.17.6",
"@types/node": "^20.17.10",
"@types/pg": "^8.11.10",
"@types/react": "^18",
"@types/react-dom": "^18",
"@types/uuid": "^10.0.0",
"autoprefixer": "^10.4.20",
"csv-parse": "^5.6.0",
"csv-parser": "^3.0.0",
"drizzle-kit": "^0.28.1",
"drizzle-orm": "^0.38.2",
"eslint": "^8",
"eslint-config-next": "15.0.3",
"fs": "^0.0.1-security",
"next-auth": "^5.0.0-beta.25",
"postcss": "^8",
"tailwindcss": "^3.4.15",

View File

@@ -1,28 +0,0 @@
import { sectionLinks } from "../../lib/linkList/sectionLinks";
import Link from "next/link";
import armoryLinks from "../../lib/linkList/sectionLinks";
let linksArray = [
sectionLinks.UPPERS,
sectionLinks.LOWERS,
sectionLinks.BARRELS,
sectionLinks.OPTICS,
sectionLinks.ACCESSORIES,
];
export const Armory = (props:any) => {
return (
(<div>
<h4>{props.titleText}</h4>
<ul>
{linksArray.map((link, index) => (
<li key={index}>
<Link href={link.URL} legacyBehavior>
{link.TEXT}
</Link>
</li>
))}
</ul>
</div>)
);
}
export default Armory;

View File

@@ -1,23 +0,0 @@
{
"name": "armory",
"version": "0.0.0",
"private": true,
"main": "./index",
"author": {
"name": "Sean Strawsburg",
"email": "sean@goforward.group",
"url": "https://goforward.group/"
},
"contributors": [
{
"name": "Don Strawsburg",
"email": "don@goforward.group",
"url": "https://goforward.group/"
},
{
"name": "Sean Strawsburg",
"email": "sean@goforward.group",
"url": "https://goforward.group/"
}
]
}

View File

@@ -1 +0,0 @@
@import '../../scss/variables.scss';

View File

@@ -1,38 +0,0 @@
import React from 'react';
import styles from './style.module.scss'
const Button = ({
children,
className,
color = 'black',
type = 'button',
...props
}) => (
<button
className={`${className} Button Button_${color}`}
type={type}
{...props}
>
{children}
</button>
);
const ButtonUnobtrusive = ({
children,
className,
type = 'button',
...props
}) => (
<button
className={`${className} Button_unobtrusive`}
type={type}
{...props}
>
{children}
</button>
);
export { ButtonUnobtrusive };
export default Button;

View File

@@ -1,23 +0,0 @@
{
"name": "button",
"version": "0.0.0",
"private": true,
"main": "./index",
"author": {
"name": "Sean Strawsburg",
"email": "sean@goforward.group",
"url": "https://goforward.group/"
},
"contributors": [
{
"name": "Don Strawsburg",
"email": "don@goforward.group",
"url": "https://goforward.group/"
},
{
"name": "Sean Strawsburg",
"email": "sean@goforward.group",
"url": "https://goforward.group/"
}
]
}

View File

@@ -1,46 +0,0 @@
.Button {
padding: 10px;
background: none;
cursor: pointer;
transition: color 0.25s ease-in-out;
transition: background 0.25s ease-in-out;
}
.Button_white {
border: 1px solid #fff;
color: #fff;
}
.Button_white:hover {
color: #000;
background: #fff;
}
.Button_black {
border: 1px solid #000;
color: #000;
}
.Button_black:hover {
color: #fff;
background: #000;
}
.Button_unobtrusive {
padding: 0;
color: #000;
background: none;
border: none;
cursor: pointer;
opacity: 1;
transition: opacity 0.25s ease-in-out;
outline: none;
}
.Button_unobtrusive:hover {
opacity: 0.35;
}
.Button_unobtrusive:focus {
outline: none;
}

View File

@@ -1,11 +0,0 @@
import React from 'react';
import styles from './style.module.css';
const ErrorMessage = ({ error }) => (
<div className="ErrorMessage">
<small>{error.toString()}</small>
</div>
);
export default ErrorMessage;

View File

@@ -1,23 +0,0 @@
{
"name": "error",
"version": "0.0.0",
"private": true,
"main": "./index",
"author": {
"name": "Sean Strawsburg",
"email": "sean@goforward.group",
"url": "https://goforward.group/"
},
"contributors": [
{
"name": "Don Strawsburg",
"email": "don@goforward.group",
"url": "https://goforward.group/"
},
{
"name": "Sean Strawsburg",
"email": "sean@goforward.group",
"url": "https://goforward.group/"
}
]
}

View File

@@ -1,5 +0,0 @@
.ErrorMessage {
margin: 20px;
display: flex;
justify-content: center;
}

View File

@@ -1,27 +0,0 @@
import { sectionLinks } from "@/app/lib/linkList/sectionLinks";
import Link from "next/link";
let linksArray = [
sectionLinks.BLOG,
sectionLinks.PRICEDROPS,
sectionLinks.BUILDS,
sectionLinks.BUILDGUIDES,
];
export const GroundZero = (props) => {
return (
(<div>
<h4>{props.titleText}</h4>
<ul>
{linksArray.map((link, index) => (
<li key={index}>
<Link href={link.URL} legacyBehavior>
{link.TEXT}
</Link>
</li>
))}
</ul>
</div>)
);
}
export default GroundZero;

View File

@@ -1 +0,0 @@
@import '../../scss/variables.scss';

View File

@@ -1,27 +0,0 @@
import { infoLinks } from "@/app/lib/linkList/infoLinks";
import Link from "next/link";
let linksArray = [
infoLinks.ABOUT,
infoLinks.FAQ,
infoLinks.DISCLOSURE,
infoLinks.PRIVACYPOLICY,
infoLinks.PIP,
infoLinks.TOS
];
export const Information = (props) => {
return (
(<div>
<h4>{props.titleText}</h4>
<ul>
{linksArray.map((link, index) => (
<li key={index}>
<Link href={link.URL} legacyBehavior>
{link.TEXT}
</Link>
</li>
))}
</ul>
</div>)
);
}
export default Information;

View File

@@ -1,23 +0,0 @@
{
"name": "information",
"version": "0.0.0",
"private": true,
"main": "./index",
"author": {
"name": "Sean Strawsburg",
"email": "sean@goforward.group",
"url": "https://goforward.group/"
},
"contributors": [
{
"name": "Don Strawsburg",
"email": "don@goforward.group",
"url": "https://goforward.group/"
},
{
"name": "Sean Strawsburg",
"email": "sean@goforward.group",
"url": "https://goforward.group/"
}
]
}

View File

@@ -1 +0,0 @@
@import '../../scss/variables.scss';

View File

@@ -1,5 +1,6 @@
import { getMags } from "@queries/PSA";
import styles from '../styles.module.css';
import PageHero from "@components/PageHero";
import SortTable from "@components/SortTable";
export const metadata = {
@@ -13,6 +14,7 @@ export default async function MagsPage() {
return (
<div>
<PageHero title="Magazines" />
<div className="container mx-auto">
<SortTable data={data}></SortTable>
</div>

View File

@@ -1,6 +1,6 @@
import { getMuzzleDevices } from "@queries/PSA";
import styles from '../styles.module.css';
import PageHero from "@components/PageHero";
import SortTable from "@components/SortTable";
export default async function MuzzleDevices() {

View File

@@ -1,5 +1,5 @@
import "../styles/globals.css";
import Navbar from "../components/Navbar";
import "@styles/globals.css";
import Navbar from "@components/Navbar";
import PopNav from "@components/PopNav/page";
import { Roboto } from 'next/font/google'
import constants from "@src/lib/constants";

View File

@@ -188,6 +188,44 @@ export const builds = pgTable("builds", {
}
});
export const bb_products = pgTable("bb_products", {
uuid: uuid().defaultRandom().primaryKey().notNull(),
upc: varchar("UPC", { length: 100 }),
sku: varchar("SKU", { length: 50 }),
manufacturerId: varchar("MANUFACTURER_ID", { length: 50 }),
brandName: varchar("BRAND_NAME", { length: 50 }),
productName: varchar("PRODUCT_NAME", { length: 255 }),
longDescription: text("LONG_DESCRIPTION"),
shortDescription: varchar("SHORT_DESCRIPTION", { length: 500 }),
department: varchar("DEPARTMENT", { length: 100 }),
category: varchar("CATEGORY", { length: 100 }),
subcategory: varchar("SUBCATEGORY", { length: 100 }),
thumbUrl: varchar("THUMB_URL", { length: 500 }),
imageUrl: varchar("IMAGE_URL", { length: 500 }),
buyLink: varchar("BUY_LINK", { length: 500 }),
keywords: varchar("KEYWORDS", { length: 500 }),
reviews: varchar("REVIEWS", { length: 500 }),
retailPrice: varchar("RETAIL_PRICE", { length: 50 }),
salePrice: varchar("SALE_PRICE", { length: 50 }),
brandPageLink: varchar("BRAND_PAGE_LINK", { length: 500 }),
brandLogoImage: varchar("BRAND_LOGO_IMAGE", { length: 500 }),
productPageViewTracking: varchar("PRODUCT_PAGE_VIEW_TRACKING", { length: 500 }),
parentGroupId: varchar("PARENT_GROUP_ID", { length: 200 }),
fineline: varchar("FINELINE", { length: 200 }),
superfineline: varchar("SUPERFINELINE", { length: 200 }),
modelnumber: varchar("MODELNUMBER", { length: 100 }),
caliber: varchar("CALIBER", { length: 200 }),
mediumImageUrl: varchar("MEDIUM_IMAGE_URL", { length: 500 }),
productContentWidget: varchar("PRODUCT_CONTENT_WIDGET", { length: 500 }),
googleCategorization: varchar("GOOGLE_CATEGORIZATION", { length: 500 }),
itemBasedCommission: varchar("ITEM_BASED_COMMISSION", { length: 500 }),
itemBasedCommissionRate: varchar("ITEM_BASED_COMMISSION RATE", { length: 50 }),
updatedAt: timestamp("updated_at", { mode: 'string' }).defaultNow().notNull(),
createdAt: timestamp("created_at", { mode: 'string' }).defaultNow().notNull(),
deletedAt: timestamp("deleted_at", { mode: 'string' }),
});
export const psa_old = pgTable("psa_old", {
sku: varchar("SKU", { length: 50 }),
manufacturerId: varchar("MANUFACTURER_ID", { length: 50 }),

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,101 @@
import * as fs from "fs";
import * as path from "path";
import { parse } from 'csv-parse';
import { bb_products } from "/home/don/code/ballistics-builder/src/drizzle/schema/schema";
import { db } from '/home/don/code/ballistics-builder/src/db/index';
type BB_PRODUCTS = {
UPC: string;
SKU: string;
'Manufacturer Id': string;
"Brand Name": string;
'Product Name': string;
'Long Description': string;
'Short Description': string;
Department: string;
Category: string;
SubCategory: string;
'Thumb URL': string;
'Image URL': string;
'Buy Link': string;
Keywords: string;
Reviews: string;
'Retail Price': string;
'Sale Price': string;
'Brand Page Link': string;
'Brand Logo Image': string;
'Product Page View Tracking': string;
'Parent Group ID': string;
Color: string;
Size: string;
Pattern: string;
Material: string;
'Age Group': string;
Gender: string;
Availability: string;
'Google Product Category': string;
'Medium Image URL': string;
'Variants XML': string;
GTIN: string;
'Key Words': string;
'Product Content Widget': string;
'Google Categorization': string;
'Item Based Commission': string;
'Item Based Commission Rate': string;
};
(() => {
const csvFilePath = path.resolve(__dirname, '/home/don/code/ballistics-builder/src/lib/dataImports/Brownells_cleaned.csv');
const headers = ['SKU', 'Manufacturer Id', 'Brand Name', 'Product Name', 'Long Description', 'Short Description', 'Department', 'Category', 'SubCategory', 'Thumb URL', 'Image URL', 'Buy Link', 'Keywords', 'Reviews', 'Retail Price', 'Sale Price', 'Brand Page Link', 'Brand Logo Image', 'Product Page View Tracking', 'Parent Group ID', 'Color', 'Size', 'Pattern', 'Material', 'Age Group', 'Gender', 'UPC', 'Availability', 'Google Product Category', 'Medium Image URL', 'Variants XML', 'GTIN', 'Key Words', 'Product Content Widget', 'Google Categorization', 'Item Based Commission', 'Item Based Commission Rate', 'Item Based Commission Rule'
];
const fileContent = fs.readFileSync(csvFilePath, { encoding: 'utf-8' });
let rows: BB_PRODUCTS[] = [];
parse(fileContent, {
delimiter: ',',
columns: headers,
from_line: 2,
}, (error, result: BB_PRODUCTS[]) => {
if (error) {
console.error(error);
}
// Insert data into the database
for (const row of result) {
db.insert(bb_products).values({
upc: row["UPC"],
sku: row.SKU,
brandName: row["Brand Name"],
productName: row["Product Name"],
longDescription: row["Long Description"],
shortDescription: row["Short Description"],
department: row.Department,
category: row.Category,
subcategory: row.SubCategory,
thumbUrl: row["Thumb URL"],
imageUrl: row["Image URL"],
buyLink: row["Buy Link"],
keywords: row.Keywords,
reviews: row.Reviews,
retailPrice: row["Retail Price"],
salePrice: row["Sale Price"],
brandPageLink: row["Brand Page Link"],
brandLogoImage: row["Brand Logo Image"],
productPageViewTracking: row["Product Page View Tracking"],
parentGroupId: row["Parent Group ID"],
mediumImageUrl: row["Medium Image URL"],
productContentWidget: row["Product Content Widget"],
googleCategorization: row["Google Categorization"],
itemBasedCommission: row["Item Based Commission"],
itemBasedCommissionRate: row["Item Based Commission Rate"]
// ...other fields from the CSV row...
}).execute();
}
});
})();

View File

@@ -1,14 +1,17 @@
{
"compilerOptions": {
"target": "ES2017",
"lib": ["dom", "dom.iterable", "esnext"],
"target": "ESNext",
"lib": [
"dom",
"dom.iterable",
"esnext"
],
"allowJs": true,
"skipLibCheck": true,
"strict": true,
"noEmit": true,
"esModuleInterop": true,
"module": "esnext",
"moduleResolution": "bundler",
"module": "commonjs",
"resolveJsonModule": true,
"isolatedModules": true,
"jsx": "preserve",
@@ -21,17 +24,23 @@
],
"paths": {
"@src/*": [
"./src/*"],
"./src/*"
],
"@/db/*": [
"./src/db/*"],
"./src/db/*"
],
"@queries/*": [
"./src/db/queries/*"],
"./src/db/queries/*"
],
"@schemas/*": [
"./src/drizzle/schema/*"],
"./src/drizzle/schema/*"
],
"@db/*": [
"./src/db/*"],
"./src/db/*"
],
"@types/*": [
"./src/types/*"],
"./src/types/*"
],
"@components/*": [
"./src/components/*",
"./src/components/ui/*",
@@ -63,8 +72,15 @@
"./src/components/Products/*"
]
},
"moduleResolution": "node"
},
"include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
"exclude": ["node_modules"]
"include": [
"next-env.d.ts",
"**/*.ts",
"**/*.tsx",
".next/types/**/*.ts"
],
"exclude": [
"node_modules"
]
}