From 6172baa9bb716179b03011d29c8d3303f71cd1aa Mon Sep 17 00:00:00 2001 From: Gabriel Dunne Date: Tue, 19 Mar 2013 01:17:07 -0700 Subject: [PATCH] adde bash script back in --- agg | 73 +++++++++++++++++++++++++++ agg.py | 147 ++++++++++++++++++++++++++++++++++++++---------------- data.json | 20 +++++++- 3 files changed, 196 insertions(+), 44 deletions(-) create mode 100755 agg diff --git a/agg b/agg new file mode 100755 index 0000000..c542c63 --- /dev/null +++ b/agg @@ -0,0 +1,73 @@ +#!/usr/bin/env bash + +ACTION=$1 +FILE=$2 +TIMESTAMP=$(date +%s) + +c_yellow='\033[0;93m' +c_green='\033[0;92m' +c_purple='\033[0;95m' +c_red='\033[0;91m' +c_blue='\033[0;94m' +c_end='\033[0m' + +S3_BUCKET='s3://agg/' +S3_URL='http://agg.s3.amazonaws.com/' +DB='agg.db' +video_folder='v/' +image_folder='i/' +declare -a video_types=('video/mp4' 'video/avi' 'video/fla'); +declare -a image_types=('image/jpg' 'image/jpeg' 'image/png' 'image/gif'); + +db_schema="CREATE TABLE IF NOT EXISTS entries(id INTEGER PRIMARY KEY AUTOINCREMENT, added DEFAULT CURRENT_TIMESTAMP, link TEXT, title TEXT, desc TEXT, name TEXT, url TEXT, mimetype TEXT)" + +if [ ! -f "$DB" ]; +then + echo -e "$c_red"No database"$c_end" + exit; +else + sqlite3 "$DB" "$db_schema" +fi + +if [ -z $ACTION ] +then + exit +fi + +if [ "dl" = $ACTION ] +then + echo "downloading $FILE" +# ~/_code/youtube-dl/youtube-dl -o "%(stitle)s-%(id)s-%(autonumber)s.%(ext)s" $FILE +fi +if [[ "ul" = $ACTION && $FILE ]] +then + + # get mime type + mimetype=$(file -b --mime-type $FILE) + echo -e $c_yellow"type\t$c_end"$mimetype + + # determine mimetype and thus the appropriate folder + case "${video_types[@]}" in *"$mimetype"*) FOLDER=$video_folder ;; esac + case "${image_types[@]}" in *"$mimetype"*) FOLDER=$image_folder ;; esac + if [ -z "$FOLDER" ]; then + echo -e $c_red"Unsupported mime-type"$c_end + exit + fi + + # get user input for data + echo -ne $c_yellow"link\t"$c_end; read -r LINK; printf -v LINK "%q" "$LINK" + echo -ne $c_yellow"title\t"$c_end; read -r TITLE; printf -v TITLE "%q" "$TITLE" + echo -ne $c_yellow"desc\t"$c_end; read -r DESC; printf -v DESC "%q" "$DESC" + URL="$S3_URL$FOLDER$FILE" + sqlite3 "$DB" "INSERT INTO entries (link, title, desc, url, name, mimetype) VALUES ('$LINK','$TITLE','$DESC', '$URL', '$FILENAME', '$mimetype')"; + + # upload file to s3 + echo -e $c_green"Uploading to s3..."$c_end + # get filename + FILENAME=$(basename $FILE) + # put file on s3 + s3cmd put --acl-public --guess-mime-type "$FILE" "$S3_BUCKET$FOLDER$FILENAME" + +fi + + diff --git a/agg.py b/agg.py index 1f33392..6237d2d 100755 --- a/agg.py +++ b/agg.py @@ -1,90 +1,153 @@ #!/usr/bin/python import json, sys, os, time -import subprocess +import subprocess, sqlite3 +import magic -yellow = '\033[0;33m' -green = '\033[0;92m' -purple = '\033[0;95m' -red = '\033[0;91m' -endc = '\033[0m' -s3_bucket="s3://agg/" +colors = {} +colors['yellow'] = '\033[0;33m' +colors['green'] = '\033[0;92m' +colors['purple'] = '\033[0;95m' +colors['red'] = '\033[0;91m' +colors['blue'] = '\033[0;94m' +colors['end'] = '\033[0m' + +video_types = [ 'video/mp4' ] +image_types = [ 'image/jpg' ] + +s3_bucket="s3://agg/" s3_url="http://agg.s3.amazonaws.com/" +db = None +_DATABASE_ = 'agg.db' + def main(): + # init DB + try: + db = sqlite3.connect(_DATABASE_) + initdb(db); + except sqlite3.Error, e: + print "Error %s:" % e.args[0] + sys.exit(1) + finally: + if db: + db.close() + + # system arguments if len(sys.argv) < 2: exit() if len(sys.argv) == 3: media_path = sys.argv[2] media_filename = os.path.split(sys.argv[2])[-1] - action = sys.argv[1] - - if action == "up": - # update data file - print green + "Uploading datafile to s3..." + endc - cmd = "s3cmd put -v --acl-public --guess-mime-type data.json " + s3_bucket + "data.json" + if action == "push-data": + # upload database + print green + "Uploading database to s3..." + colors['end'] + cmd = ( "s3cmd put -v --acl-public --guess-mime-type agg.db " + + s3_bucket + _DATABASE_ ) print cmd os.system(cmd); - # upload media - elif action == "ul": + elif action == "pull-data": + # download database + print colors['green'] + "Downloading database from s3..." + colors['end'] + cmd = "s3cmd get --force " + s3_bucket + _DATABASE_ + print cmd + os.system(cmd); + + elif action == "ul" : + # upload media + + + # determine file type and distinguish if it's an image or a video + mime = magic.from_file(media_path, mime=True) + print colors['yellow'] + "type\t" + colors['end'] + mime + if (mime in video_types): + folder = "v/" + elif (mime in image_types): + folder = "i/" + else: + print colors['red'] + "File is unsupported mime-type" + colors['end'] + # check to see if media file already exists - p = subprocess.Popen("s3cmd ls " + s3_bucket + "v/" + media_filename, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + p = subprocess.Popen("s3cmd ls " + s3_bucket + folder + media_filename, + shell=True, stdout = subprocess.PIPE, stderr = subprocess.STDOUT) for line in p.stdout.readlines(): print line, - print red + "File exists." + endc + print colors['red'] + "File exists." + colors['end'] exit() retval = p.wait() - # start new entry - new_entry = get_new_entry(media_filename) - - print green + "Uploading to s3..." + endc - cmd = "s3cmd put --acl-public --guess-mime-type " + media_path + " " + s3_bucket + "v/" + media_filename - print cmd - os.system(cmd); - print green + "Downloading data file..." + endc - cmd = "s3cmd get --force " + s3_bucket + "data.json" + # start new entry + new_entry = {} + new_entry['link'] = get_user_input("link ") + new_entry['title'] = get_user_input("title ") + new_entry['desc'] = get_user_input("desc ") + new_entry['name'] = media_filename + new_entry['url'] = s3_url + "v/" + media_filename + new_entry['mime'] = mime + + + print colors['green'] + "Entering data in db" + colors['end'] + try: + db = sqlite3.connect(_DATABASE_) + cur = db.cursor() + cur.execute("INSERT INTO entries VALUES(NULL, DATETIME('NOW'), ?, ?, ?, ?, ?, ?)", ( new_entry['link'], new_entry['title'], new_entry['desc'], new_entry['name'], new_entry['mime'], new_entry['url'])) + db.commit() + data = cur.fetchone() + print data + except sqlite3.Error, e: + print "Error %s:" % e.args[0] + sys.exit(1) + finally: + if db: + db.close() + + print colors['green'] + "Uploading to s3..." + colors['end'] + cmd = "s3cmd put --acl-public --guess-mime-type " + media_path + " " + s3_bucket + folder + media_filename print cmd os.system(cmd); - with open('data.json', 'r') as f: - data = json.load(f) - data['entries'].append(new_entry) - with open('data.json', 'w') as f: - json.dump(data, f) + #print colors['purple'] + json.dumps(new_entry, indent = 2) + colors['end'] - print purple + json.dumps(new_entry, indent = 2) + endc # download media via youtube-dl elif action == "dl": print "download" - +def initdb(db): + cur = db.cursor() + cur.execute("CREATE TABLE IF NOT EXISTS entries(" + + "_id integer primary key autoincrement, " + + "created date default CURRENT_DATE, " + + "link TEXT, " + + "title TEXT, " + + "desc TEXT, " + + "name TEXT, " + + "url TEXT, " + + "mime TEXT" + + ")") -def get_new_entry(filename): - new_entry = { "filename" : filename, "added" : int(time.time()) } - new_entry['url'] = s3_url + "v/" + filename - new_entry['link'] = get_user_input("link ") - new_entry['title'] = get_user_input("title ") - new_entry['desc'] = get_user_input("desc ") - return new_entry + +def sqliteVersion(db): + cur = db.cursor() + cur.execute('SELECT SQLITE_VERSION()') + data = cur.fetchone() + print "SQLite version: %s" % data def get_user_input(question): while True : - sys.stdout.write(yellow + question + "\t" + endc) + sys.stdout.write(colors['yellow'] + question + "\t" + colors['end']) choice = raw_input() return choice - if __name__ == "__main__": main() diff --git a/data.json b/data.json index a61b697..d475e96 100644 --- a/data.json +++ b/data.json @@ -1,3 +1,19 @@ { - "entries": [] -} + "entries": [ + { + "added": 1363583266, + "desc": "", + "filename": "yeah-lamb.mp4", + "link": "", + "title": "yeah lamb", + "url": "http://agg.s3.amazonaws.com/v/yeah-lamb.mp4" + }, + { + "added": 1363583344, + "desc": "", + "filename": "BmpMPZCgSvs.mp4", + "link": "", + "title": "cymatics", + "url": "http://agg.s3.amazonaws.com/v/BmpMPZCgSvs.mp4" + } + ] -- 2.34.1