设为首页 加入收藏

TOP

Python模块化开发组织代码程序示例(一)
2017-01-24 08:15:36 】 浏览:898
Tags:Python 模块化 开发 组织 代码 程序 示例

样例包含三部分代码,周的处理函数部分、业务数据处理部分及多线程跑批调度处理部分。


代码按功能分类存放,有助于使代码更清晰,通过from...import的方式,使代码重复使用。


另外,多线程的调用部分,有效处理了程序先后依赖及多程序串并行跑批问题,为以后相似问题的处理,提供了借鉴。


1、周处理函数


/Users/nisj/PycharmProjects/EsDataProc/Remain_Data_Proc/WeekCalc.py


# -*- coding=utf-8 -*-
import warnings
import datetime


warnings.filterwarnings("ignore")


def getNowYearWeek():
? ? # 当前时间年第几周的计算
? ? timenow = datetime.datetime.now() - datetime.timedelta(days=7)
? ? NowYearWeek = timenow.isocalendar()
? ? return str(NowYearWeek[0])+"#"+str(NowYearWeek[1])


def dateRange(beginDate, endDate):
? ? dates = []
? ? dt = datetime.datetime.strptime(beginDate, "%Y-%m-%d")
? ? date = beginDate[:]
? ? while date <= endDate:
? ? ? ? dates.append(date)
? ? ? ? dt = dt + datetime.timedelta(1)
? ? ? ? date = dt.strftime("%Y-%m-%d")
? ? return dates


def weekRang(beginDate, endDate):
? ? week = set()
? ? for date in dateRange(beginDate, endDate):
? ? ? ? week.add(datetime.date(int(date[0:4]), int(date[5:7]), int(date[8:10])).isocalendar()[0:2])


? ? wk_l = []
? ? for wl in sorted(list(week)):
? ? ? ? wk_l.append(str(wl[0])+'#'+str(wl[1]))
? ? return wk_l


def currWeekList(his_week):
? ? last_wk = datetime.datetime.now() - datetime.timedelta(days=7)
? ? end_day = str(last_wk)[0:10]
? ? curr_week_list = []
? ? for week in weekRang('2015-07-01', end_day):
? ? ? ? if (int(week[0:4]) == int(his_week[0:4]) and int(week[5:]) >= int(his_week[5:])) or (int(week[0:4]) > int(his_week[0:4])):
? ? ? ? ? ? curr_week_list.append(week)
? ? return curr_week_list


def hisRunWeekList(his_week):
? ? batch_week_list = []
? ? for curr_week in currWeekList(his_week):
? ? ? ? if (int(his_week[0:4]) == int(curr_week[0:4]) and int(his_week[5:]) <= int(curr_week[5:])) or (int(his_week[0:4]) < int(curr_week[0:4])):
? ? ? ? ? ? batch_week_list.append(([curr_week, his_week],None))
? ? return batch_week_list


def RuningWeekList():
? ? curr_week = getNowYearWeek()
? ? batch_week_list = []
? ? for his_week in currWeekList('2015#27'):
? ? ? ? ? ? if (int(his_week[0:4]) == int(curr_week[0:4]) and int(his_week[5:]) <= int(curr_week[5:])) or (int(his_week[0:4]) < int(curr_week[0:4])):
? ? ? ? ? ? ? ? batch_week_list.append(([curr_week, his_week],None))
? ? return batch_week_list


def getWeekFristday(weekflag):
? ? yearnum = weekflag[0:4]? # 取到年份
? ? weeknum = weekflag[5:7]? # 取到周
? ? stryearstart = yearnum + '0101'? # 当年第一天
? ? yearstart = datetime.datetime.strptime(stryearstart, '%Y%m%d')? # 格式化为日期格式
? ? yearstartcalendarmsg = yearstart.isocalendar()? # 当年第一天的周信息
? ? yearstartweekday = yearstartcalendarmsg[2]
? ? yearstartyear = yearstartcalendarmsg[0]
? ? if yearstartyear < int(yearnum):
? ? ? ? daydelat = (8 - int(yearstartweekday)) + (int(weeknum) - 1) * 7
? ? else:
? ? ? ? daydelat = (8 - int(yearstartweekday)) + (int(weeknum) - 2) * 7


? ? week1day = (yearstart + datetime.timedelta(days=daydelat)).date()
? ? return week1day


# Batch Test
# his_week_list = ['2015#46', '2015#45', '2016#2']
# batch_week_list = []
# for his_week in his_week_list:
#? ? batch_week_list.extend(hisRunWeekList(his_week))
# print batch_week_list
# print getWeekFristday('2016#11')
# his_week = '2016#11'
# print currWeekList(his_week)


# print getNowYearWeek()


2、业务处理部分
/Users/nisj/PycharmProjects/EsDataProc/Remain_Data_Proc/Hive_remain_byWeek_proc.py


# -*- coding=utf-8 -*-
import time
import os
import re
from WeekCalc import *


warnings.filterwarnings("ignore")



def newuser_byweek_proc(batch_week):
? ? week1day = getWeekFristday(batch_week)
? ? os.system("""/usr/lib/hive-current/bin/hive -e " \
? ? alter table bi_newuser_byweek drop if exists partition(pt_week='%s'); \
? ? al

首页 上一页 1 2 3 4 5 下一页 尾页 1/5/5
】【打印繁体】【投稿】【收藏】 【推荐】【举报】【评论】 【关闭】 【返回顶部
上一篇用Python将统计数据不存在的记录.. 下一篇使用 Java 命令编译运行 Java 程序

最新文章

热门文章

Hot 文章

Python

C 语言

C++基础

大数据基础

linux编程基础

C/C++面试题目