import gzip
import random
import csv

def parse(path):
  g = gzip.open(path, 'r')
  for l in g:
    yield eval(l)

meta = {}

def split(review):
  review = review.replace('?', '?.')
  review = review.replace('!', '!.')
  review = [l for l in review.split('.') if len(l) > 10]
  return review

for m in parse("meta_Clothing_Shoes_and_Jewelry.json.gz"):
  meta[m['asin']] = m

csvOut = open("outDescriptions.csv", 'w')
writer = csv.writer(csvOut, delimiter = '\t')
nw = 0

mks = meta.keys()
random.shuffle(mks)

for k in mks:
  m = meta[k]
  if nw < 500:
    try:
      front = [m['asin'], "http://amazon.com/dp/" + m['asin'], m['title'], m['imUrl'], '/'.join(m['categories'][0])]
      sp = split(m['description'])
      if len(sp) < 3:
        continue
      for line in sp:
        writer.writerow(front + [line])
        front = ['' for w in front]
      nw += 1
    except Exception as e:
      print e




reviews = []

for r in parse("reviews_Clothing_Shoes_and_Jewelry.json.gz"):
  if not meta.has_key(r['asin']):
    continue
  m = meta[r['asin']]
  if not m.has_key('imUrl'):
    continue
  if not m.has_key('title'):
    continue
  if not m.has_key('categories'):
    continue
  #if (random.random() > 1.0/10000): continue
  isJ = False
  for cc in m['categories']:
    for c in cc[2:]:
      if "Jewelry" in c:
        isJ = True
  if (isJ): reviews.append(r)

random.shuffle(reviews)

csvOut = open("outJewelry.csv", 'w')
writer = csv.writer(csvOut, delimiter = '\t')



for r in reviews:
  m = meta[r['asin']]
  front = [r['asin'], "http://amazon.com/dp/" + r['asin'], m['title'], m['imUrl'], '/'.join(m['categories'][0])]
  sp = split(r['reviewText'])
  if len(sp) < 3:
    continue
  for line in sp:
    writer.writerow(front + [line])
    front = ['' for w in front]