From 9d7fea9515fdd0c68950eae7b12f34e790f2d6cf Mon Sep 17 00:00:00 2001 From: josch Date: Wed, 9 Apr 2008 12:23:14 +0000 Subject: [PATCH] restructured upload process git-svn-id: http://yolanda.mister-muffin.de/svn@286 7eef14d0-6ed0-489d-bf55-20463b2d70db --- trunk/include.pl | 1 + trunk/init_sql.pl | 42 ++++----- trunk/locale/en-us.xml | 4 + trunk/tools/daemon.pl | 191 +++++++---------------------------------- trunk/uploader.pl | 112 +++++++++++++++++++++++- 5 files changed, 168 insertions(+), 182 deletions(-) diff --git a/trunk/include.pl b/trunk/include.pl index f729d05..81a770e 100644 --- a/trunk/include.pl +++ b/trunk/include.pl @@ -8,6 +8,7 @@ use Digest::SHA qw(sha256_hex); use LWPx::ParanoidAgent; use Net::OpenID::Consumer; use CGI::Carp qw(fatalsToBrowser set_message); +use File::Copy; set_message("It's not a bug, it's a feature!!
(include this error message in your bugreport here: Yolanda bugtracker)"); diff --git a/trunk/init_sql.pl b/trunk/init_sql.pl index 860f13a..75e6daf 100644 --- a/trunk/init_sql.pl +++ b/trunk/init_sql.pl @@ -33,15 +33,6 @@ $dbh->do(qq{insert into ) }) or die $dbh->errstr; -$dbh->do(qq{create table - config - ( - attribute varchar(255) not null, - value varchar(255) not null, - primary key (attribute) - ) -}) or die $dbh->errstr; - $dbh->do(qq{create table users ( @@ -72,20 +63,25 @@ $dbh->do(qq{insert into $dbh->do(qq{create table uploaded ( - id int auto_increment not null, - title varchar(255) not null, - description text not null, - userid int not null, - timestamp bigint not null, - creator varchar(255) not null, - subject varchar(255) not null, - source varchar(255) not null, - language varchar(255) not null, - coverage varchar(255) not null, - rights varchar(255) not null, - license varchar(255) not null, - status int default 0, - primary key (id) + id int auto_increment not null, + title varchar(255) not null, + description text not null, + userid int not null, + timestamp bigint not null, + creator varchar(255) not null, + subject varchar(255) not null, + source varchar(255) not null, + language varchar(255) not null, + coverage varchar(255) not null, + rights varchar(255) not null, + license varchar(255) not null, + filesize int not null, + duration int not null, + width smallint not null, + height smallint not null, + fps float not null, + hash char(64) not null, + primary key (id) ) }) or die $dbh->errstr; diff --git a/trunk/locale/en-us.xml b/trunk/locale/en-us.xml index 269872a..931d47d 100755 --- a/trunk/locale/en-us.xml +++ b/trunk/locale/en-us.xml @@ -65,6 +65,10 @@ Invalid URL URL provided doesn't seem to have a head tag Error fetching the provided URL + Invalid audio and/or video stream + Cannot read video file + File is not a video + Video has already been uploaded diff --git a/trunk/tools/daemon.pl b/trunk/tools/daemon.pl index 1e32453..fbf6c3c 100755 --- a/trunk/tools/daemon.pl +++ b/trunk/tools/daemon.pl @@ -39,174 +39,49 @@ sub interrupt $dbh = DBI->connect("DBI:mysql:$database:$dbhost", $dbuser, $dbpass) or interrupt "could not connect to db"; -#video status: -# 0 - new entry - nothing done yet -# 1 - valid public video -# 2 - error: invalid audio and/or video stream -# 3 - error: file not found -# 4 - error: file is not a video -# 5 - error: video is a duplicate - while(1) { #get fresh video id from db - my $sth = $dbh->prepare(qq{select id from uploaded where status = 0 limit 1}) or interrupt $dbh->errstr; + my $sth = $dbh->prepare(qq{select id, filesize, duration, width, height, fps, hash + from uploaded where duration != 0 and width != 0 and height != 0 limit 1}) or interrupt $dbh->errstr; $sth->execute() or interrupt $dbh->errstr; - my ($id) = $sth->fetchrow_array(); + my ($id,$filesize, $duration, $width, $height, $fps, $sha) = $sth->fetchrow_array(); $sth->finish() or interrupt $dbh->errstr; if($id) { - $info = `export SDL_VIDEODRIVER="dummy"; ffplay -stats -an -vn -nodisp /tmp/$id 2>&1`; + $vmaxheight = 640; - if($info =~ /ignoring/) - { - appendlog "id: $id", - "error: invalid stream", - "ffplay msg: $info"; - - #write status 2 to uploaded table - $dbh->do(qq{update uploaded set status = ? where id = ?}, undef, 2, $id) or interrupt $dbh->errstr; - unlink "/tmp/$id"; - } - elsif ($info =~ /I\/O error occured/) - { - appendlog "id: $id", - "error: file not found", - "ffplay msg: $info"; - - #write status 3 to uploaded table - $dbh->do(qq{update uploaded set status = ? where id = ?}, undef, 3, $id) or interrupt $dbh->errstr; - unlink "/tmp/$id"; - } - elsif ($info =~ /Unknown format/ or $info =~ /could not find codec parameters/) - { - appendlog "id: $id", - "error: file is of unknown format", - "ffplay msg: $info"; - - #write status 4 to uploaded table - $dbh->do(qq{update uploaded set status = ? where id = ?}, undef, 4, $id) or interrupt $dbh->errstr; - unlink "/tmp/$id"; - } - else - { - $sha = new Digest::SHA(256); - $sha->addfile("/tmp/$id"); - $sha = $sha->hexdigest; - - #check if this hash is already in database - my $sth = $dbh->prepare(qq{select id from videos where hash = ? limit 1}) or interrupt $dbh->errstr; - $sth->execute($sha) or interrupt $dbh->errstr; - my ($resultid) = $sth->fetchrow_array(); - $sth->finish() or interrupt $dbh->errstr; - - #if so, then video is a duplicate (alternatively ALL HAIL QUANTUM COMPUTING) - if($resultid) - { - appendlog "id: $id", - "error: video already uploaded: $resultid"; - - #write status 5 to uploaded table - $dbh->do(qq{update uploaded set status = ? where id = ?}, undef, 5, $id) or interrupt $dbh->errstr; - unlink "/tmp/$id"; - } - else - { - ($container, $duration) = $info =~ /Input \#0, (\w+),.+?\n.+?Duration: (\d{2}:\d{2}:\d{2}\.\d)/; - - #these two regexes have to be applied seperately because nobody knows which stream (audio or video) comes first - ($audio) = $info =~ /Audio: (\w+)/; - ($video, $width, $height, $fps) = $info =~ /Video: ([\w\d]+),.+?(\d+)x(\d+),.+?(\d+\.\d+) fps/; - - if(!$audio or !$video or !$duration) - { - appendlog "id: $id", - "error: error: stream is missing or video is corrupt", - "audio: $audio", - "video: $video", - "duration: $duration", - "ffplay msg: $info"; - - #write status 2 to uploaded table - $dbh->do(qq{update uploaded set status = ? where id = ?}, undef, 2, $id) or interrupt $dbh->errstr; - unlink "/tmp/$id"; - } - else - { - $filesize = -s "/tmp/$id"; - - #convert hh:mm:ss.s duration to full seconds - thanks perl for making this so damn easy! - #don't want to know how this would look in python or php... hell I don't even have to create extra variables! - $duration =~ /^(\d{2}):(\d{2}):(\d{2})\.(\d)$/; - $duration = int($1*3600 + $2*60 + $3 + $4/10 + .5); - - #create thumbnail - $thumbnailsec = int(rand($duration)); - $previewsec = $thumbnailsec; - - #the width/height calculation could of course be much shorter but less readable then - #all thumbs have equal height - $tnmaxheight = 120; - $tnheight = $tnmaxheight; - $tnwidth = int($tnheight*($width/$height)/2 + .5)*2; - - system "ffmpeg -i /tmp/$id -vcodec mjpeg -vframes 1 -an -f rawvideo -ss $thumbnailsec -s ".$tnwidth."x$tnheight $root/video-stills/thumbnails/$id"; - system "ffmpeg -i /tmp/$id -vcodec mjpeg -vframes 1 -an -f rawvideo -ss $previewsec $root/video-stills/previews/$id"; - - $vmaxheight = 640; - - #check if the upload already is in the right format and smaller/equal max-width/height - if ($container eq 'ogg' and $video eq 'theora' and $audio eq 'vorbis' and $height <= $vmaxheight) - { - appendlog $id, "file already is ogg-theora/vorbis"; - - #add video to videos table - $dbh->do(qq{insert into videos select id, title, description, userid, timestamp, creator, - subject, source, language, coverage, rights, license, ?, ?, ?, ?, ?, ?, 0, 0 - from uploaded where id = ?}, undef, $filesize, $duration, $width, - $height, $fps, $sha, $id) or interrupt $dbh->errstr; - - #move video - move "/tmp/$id", "$root/videos/$id"; - } - else #encode video - { - #video height is either ther maximum video height - #or when the original is smaller than that the original height - #check for multiple by 8 - $vheight = $vmaxheight <= $height ? $vmaxheight : int($height/8 + .5)*8; - $vwidth = int($vheight*($width/$height)/8 + .5)*8; - - $abitrate = 64; - $vbitrate = int(($filesize*8) / $duration + .5) - $abitrate; - - #TODO: add metadata information - system "ffmpeg2theora --optimize --videobitrate $vbitrate --audiobitrate $abitrate --sharpness 0 --width $vwidth --height $vheight --output $root/videos/$id /tmp/$id"; - - appendlog $id, $audio, $video, $vwidth, $vheight, $fps, $duration, $sha; - - $filesize = -s "$root/videos/$id"; - - #add video to videos table - $dbh->do(qq{insert into videos select id, title, description, userid, timestamp, creator, - subject, source, language, coverage, rights, license, ?, ?, ?, ?, ?, ?, 0, 0 - from uploaded where id = ?}, undef, $filesize, $duration, $vwidth, - $vheight, $fps, $sha, $id) or interrupt $dbh->errstr; - - #delete temp file - unlink "/tmp/$id"; - } - - #create torrent file - - - #delete from uploaded table - $dbh->do(qq{delete from uploaded where id = ?}, undef, $id) or interrupt $dbh->errstr; - } - } - } + #video height is either the maximum video height + #or (when the original is smaller than that) the original height + #check for multiple by 8 + $vheight = $vmaxheight <= $height ? $vmaxheight : int($height/8 + .5)*8; + $vwidth = int($vheight*($width/$height)/8 + .5)*8; + + $abitrate = 64; + $vbitrate = int(($filesize*8) / $duration + .5) - $abitrate; + + #TODO: add metadata information + system "ffmpeg2theora --optimize --videobitrate $vbitrate --audiobitrate $abitrate --sharpness 0 --width $vwidth --height $vheight --output $root/videos/$id /tmp/$id"; + + appendlog $id, $audio, $video, $vwidth, $vheight, $fps, $duration, $sha; + + $filesize = -s "$root/videos/$id"; + + #add video to videos table + $dbh->do(qq{insert into videos select id, title, description, userid, timestamp, creator, + subject, source, language, coverage, rights, license, ?, duration, ?, ?, fps, hash, 0, 0 + from uploaded where id = ?}, undef, $filesize, $vwidth, + $vheight, $id) or interrupt $dbh->errstr; + + #delete temp file + unlink "/tmp/$id"; + + #TODO:create torrent file + + #delete from uploaded table + $dbh->do(qq{delete from uploaded where id = ?}, undef, $id) or interrupt $dbh->errstr; } else { diff --git a/trunk/uploader.pl b/trunk/uploader.pl index 4b63dae..f58f6d5 100644 --- a/trunk/uploader.pl +++ b/trunk/uploader.pl @@ -48,7 +48,117 @@ if($userinfo->{'id'} && $query->param("DC.Title") && } close TEMPFILE; - print $query->redirect("index.pl?information=information_uploaded&value=$domain/video/".urlencode($query->param("DC.Title"))."/$id/"); + $info = `export SDL_VIDEODRIVER="dummy"; ffplay -stats -an -vn -nodisp /tmp/$id 2>&1`; + + if($info =~ /ignoring/) + { + #delete from uploaded table + $dbh->do(qq{delete from uploaded where id = ?}, undef, $id) or die $dbh->errstr; + unlink "/tmp/$id"; + print $query->redirect("index.pl?error=error_upload_invalid_stream"); + } + elsif ($info =~ /I\/O error occured/) + { + #delete from uploaded table + $dbh->do(qq{delete from uploaded where id = ?}, undef, $id) or die $dbh->errstr; + unlink "/tmp/$id"; + print $query->redirect("index.pl?error=error_upload_io"); + } + elsif ($info =~ /Unknown format/ or $info =~ /could not find codec parameters/) + { + #delete from uploaded table + $dbh->do(qq{delete from uploaded where id = ?}, undef, $id) or die $dbh->errstr; + unlink "/tmp/$id"; + print $query->redirect("index.pl?error=error_upload_not_a_video"); + } + else + { + $sha = new Digest::SHA(256); + $sha->addfile("/tmp/$id"); + $sha = $sha->hexdigest; + + #check if this hash is already in database + my $sth = $dbh->prepare(qq{select id from videos where hash = ? limit 1}) or die $dbh->errstr; + $sth->execute($sha) or die $dbh->errstr; + my ($resultid) = $sth->fetchrow_array(); + $sth->finish() or die $dbh->errstr; + + #if so, then video is a duplicate (alternatively ALL HAIL QUANTUM COMPUTING) + if($resultid) + { + #delete from uploaded table + $dbh->do(qq{delete from uploaded where id = ?}, undef, $id) or die $dbh->errstr; + unlink "/tmp/$id"; + print $query->redirect("index.pl?error=error_upload_duplicate"); + } + else + { + ($container, $duration) = $info =~ /Input \#0, (\w+),.+?\n.+?Duration: (\d{2}:\d{2}:\d{2}\.\d)/; + + #these two regexes have to be applied seperately because nobody knows which stream (audio or video) comes first + ($audio) = $info =~ /Audio: (\w+)/; + ($video, $width, $height, $fps) = $info =~ /Video: ([\w\d]+),.+?(\d+)x(\d+),.+?(\d+\.\d+) fps/; + + if(!$video or !$duration) + { + #delete from uploaded table + $dbh->do(qq{delete from uploaded where id = ?}, undef, $id) or die $dbh->errstr; + unlink "/tmp/$id"; + print $query->redirect("index.pl?error=error_upload_invalid_stream"); + } + else + { + $filesize = -s "/tmp/$id"; + + #convert hh:mm:ss.s duration to full seconds - thanks perl for making this so damn easy! + #don't want to know how this would look in python or php... hell I don't even have to create extra variables! + $duration =~ /^(\d{2}):(\d{2}):(\d{2})\.(\d)$/; + $duration = int($1*3600 + $2*60 + $3 + $4/10 + .5); + + #create thumbnail + $thumbnailsec = int(rand($duration)); + $previewsec = $thumbnailsec; + + #the width/height calculation could of course be much shorter but less readable then + #all thumbs have equal height + $tnmaxheight = 120; + $tnheight = $tnmaxheight; + $tnwidth = int($tnheight*($width/$height)/2 + .5)*2; + + system "ffmpeg -i /tmp/$id -vcodec mjpeg -vframes 1 -an -f rawvideo -ss $thumbnailsec -s ".$tnwidth."x$tnheight $root/video-stills/thumbnails/$id"; + system "ffmpeg -i /tmp/$id -vcodec mjpeg -vframes 1 -an -f rawvideo -ss $previewsec $root/video-stills/previews/$id"; + + $vmaxheight = 640; + + #check if the upload already is in the right format and smaller/equal max-width/height + if ($container eq 'ogg' and $video eq 'theora' and ($audio eq 'vorbis' or not $audio) and $height <= $vmaxheight) + { + #add video to videos table + $dbh->do(qq{insert into videos select id, title, description, userid, timestamp, creator, + subject, source, language, coverage, rights, license, ?, ?, ?, ?, ?, ?, 0, 0 + from uploaded where id = ?}, undef, $filesize, $duration, $width, + $height, $fps, $sha, $id) or die $dbh->errstr; + + #delete from uploaded table + $dbh->do(qq{delete from uploaded where id = ?}, undef, $id) or die $dbh->errstr; + + move("/tmp/$id", "$root/videos/$id"); + + #TODO:create torrent file + } + else + { + #write all valueable information to database so the daemon can fetch it + $dbh->do(qq{update uploaded set filesize = ?, duration = ?, width = ?, + height = ?, fps = ?, hash = ? where id = ?}, undef, $filesize, $duration, $width, + $height, $fps, $sha, $id) or die $dbh->errstr; + } + + #print success to the user + print $query->redirect("index.pl?information=information_uploaded&value=$domain/video/".urlencode($query->param("DC.Title"))."/$id/"); + } + } + } } else {