from bs4 import BeautifulSoup
from os.path import expanduser
import os
from tqdm import tqdm
import pandas as pd
import requests
import urllib2
import time
import lxml

BASEURL = "https://www.cia.gov"
PAGINATE_PATH ="/library/readingroom/collection/scientific-abstracts?page="
PDF_DIR = expanduser("~") + "/cia_pdfs/"
RANGE = 1654 # pages, of 20 docs per page
TEST_RANGE = 10
SKIPPED_FILES = []

def retrieve_file(url, name):
  filepath = PDF_DIR + name + ".pdf"
  if not os.path.exists(PDF_DIR):
    os.makedirs(PDF_DIR)

    try:
      response = urllib2.urlopen(url)
      with open(filepath, 'w+') as f:
        f.write(response.read())
        f.close()
    except urllib2.URLError as e:
      print ('WiFi connection perhaps lost !! Trying one more time...')
      try:
        response = urllib2.urlopen(url)
        with open(filepath, 'w+') as f:
          f.write(response.read())
          f.close()
      except:
        print ('WiFi connection really lost !! Bailing out..')
        print (e)
        SKIPPED_FILES.append(name)

def meta_df(download_bool):
  df = pd.DataFrame(columns=['id','title','classification','publication_date'])
  idx = 0

  for i in tqdm(xrange(TEST_RANGE)):

    pagination_link = BASEURL + PAGINATE_PATH + str(i)
    pagination_page = requests.get(pagination_link)
    p_soup = BeautifulSoup(pagination_page.content, 'lxml')

    for doc_title in p_soup.find_all("h4", class_="field-content"):

      a = doc_title.select_one("a")
      link = str(a.get('href'))
      TITLE = str(a.string or "")

      doc_page = requests.get(BASEURL + link)
      m_soup = BeautifulSoup(doc_page.content, 'lxml')
      time.sleep (.05)

      try:
        PUB_DATE = m_soup.select_one(".field-name-field-pub-date").select_one("span").get('content')
      except:
        PUB_DATE = ""
      try:
        ID = m_soup.select_one(".field-name-field-document-number").select_one(".field-item").string
      except:
        ID = ""
      try:
        CLASSIFICATION = m_soup.select_one(".field-name-field-original-classification").select_one(".field-item").string
      except:
        CLASSIFICATION = ""

      if download_bool:
        PDF = m_soup.select_one(".file").select_one("a").get('href')
        retrieve_file(PDF, ID)

    df.loc[idx] = [ID, TITLE, CLASSIFICATION, PUB_DATE]
    idx+=1  

  return df

df = meta_df(True)