# -*- coding: utf-8 -*-

import codecs
import os
import torndb
import arrow

def create_data(db):
    # 创建分类和标签
    categories = db.query("select type, slug, name from typecho_metas")
    for cate in categories:
        path = 'data/%s' % cate.slug
        if not os.path.exists(path):
            os.makedirs(path)
        f = codecs.open('%s/index.md' % path, 'w', "utf-8")
        f.write("title: %s\n" % cate.slug)
        f.write("date: %s\n" % arrow.now().format('YYYY-MM-DD HH:mm:ss'))
        # 区分分类和标签
        if cate.type == 'category':
            f.write('type: "categories"\n')
        elif cate.type == 'tags':
            f.write('type: "tags"\n')
        # 禁止评论
        f.write("comments: true\n")
        f.write("---\n")
        f.close()

    # 创建文章
    entries = db.query("select cid, title, slug, text, created from typecho_contents where type='post'")
    for e in entries:
        title = e.title
        urlname = e.slug
        print(title)
        content = str(e.text).replace('<!--markdown-->', '')
        tags = []
        category = ""
        # 找出文章的tag及category
        metas = db.query(
            "select type, name, slug from `typecho_relationships` ts, typecho_metas tm where tm.mid = ts.mid and ts.cid = %s",
            e.cid)
        for m in metas:
            if m.type == 'tag':
                tags.append(m.name)
            if m.type == 'category':
                category = m.slug
        path = 'data/_posts/'
        if not os.path.exists(path):
            os.makedirs(path)
        f = codecs.open(r"%s%s.md" % (path,title), 'w', "utf-8")
        f.write("---\n")
        f.write("title: %s\n" % title)
        f.write("date: %s\n" % arrow.get(e.created).format('YYYY-MM-DD HH:mm:ss'))
        f.write("categories: %s\n" % category)
        f.write("tags: [%s]\n" % ','.join(tags))
        f.write("urlname: %s\n" % urlname)
        f.write("---\n")
        f.write(content)
        f.close()


def main():
    # 把数据库连接信息
    db = torndb.Connection(host="******", database="******", user="******", password="******")
    create_data(db)

if __name__ == "__main__":
    main()

# encoding=utf-8

import requests
from bs4 import BeautifulSoup
import csv
import codecs

DOWNLOAD_URL = 'http://movie.douban.com/top250'


def download_page(url):
    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0'}
    data = requests.get(url, headers=headers).content
    return data


def parse_html(html):
    #更换解析器，详情 https://www.crummy.com/software/BeautifulSoup/bs4/doc/index.zh.html#id9
    soup = BeautifulSoup(html, "lxml")
    movies_list_soup = soup.find('ol', attrs={'class': 'grid_view'})
    movie_name_list = []
    movie_rating_num_list = []
    movie_quote_list = []

    for movies_li in movies_list_soup.find_all('li'):
        movie_name = movies_li.find('div', attrs={'class': 'hd'}).find('span', attrs={'class': 'title'}).getText()
        movie_rating = movies_li.find('div', attrs={'class': 'bd'}).find('span', attrs={'class': 'rating_num',
                                                                                        'property': 'v:average'}).getText()
        #movie_quote = movies_li.find('div', attrs={'class': 'bd'}).find('span', attrs={'class': 'inq'}).getText()
        movie_quote_obj = movies_li.find('div', attrs={'class': 'bd'}).find('span', attrs={'class': 'inq'})
        ### 发现某部电影评价为空，此处判断评价是否为空
        if movie_quote_obj is None:
            movie_quote = ''
        else:
            movie_quote = movie_quote_obj.getText()

        movie_name_list.append(movie_name)
        movie_rating_num_list.append(movie_rating)
        movie_quote_list.append(movie_quote)

    next_page = soup.find('span', attrs={'class': 'next'}).find('a')
    if next_page:
        return movie_name_list, movie_rating_num_list, movie_quote_list, DOWNLOAD_URL + next_page['href']

    return movie_name_list, movie_rating_num_list, movie_quote_list, None


def main():
    url = DOWNLOAD_URL
    #utf_8_sig 解决windows下csv文件乱码问题
    with codecs.open('movies_top_250.csv', 'wb', encoding='utf_8_sig') as csvFile:
        writer = csv.writer(csvFile)
        writer.writerow((u'电影名', u'评分', u'理由'))
        while url:
            html = download_page(url)
            movies, rating, quote, url = parse_html(html)
            for i in range(len(movies)):
                writer.writerow((movies[i], rating[i], quote[i]))
                
if __name__ == '__main__':
    main()