josch/yolanda
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

restructured upload process

Browse source 
git-svn-id: http://yolanda.mister-muffin.de/svn@286 7eef14d0-6ed0-489d-bf55-20463b2d70db
This commit is contained in:
josch 2008-04-09 12:23:14 +00:00
parent a5e3e3f198
commit 9d7fea9515
5 changed files with 168 additions and 182 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
trunk/include.pl
View file

@ -8,6 +8,7 @@ use Digest::SHA qw(sha256_hex);
use LWPx::ParanoidAgent; use LWPx::ParanoidAgent;
use Net::OpenID::Consumer; use Net::OpenID::Consumer;
use CGI::Carp qw(fatalsToBrowser set_message); use CGI::Carp qw(fatalsToBrowser set_message);
use File::Copy;
set_message("It's not a bug, it's a feature!!<br />(include this error message in your bugreport here: <a href=\"http://yolanda.mister-muffin.de/newticket\">Yolanda bugtracker</a>)"); set_message("It's not a bug, it's a feature!!<br />(include this error message in your bugreport here: <a href=\"http://yolanda.mister-muffin.de/newticket\">Yolanda bugtracker</a>)");

16
trunk/init_sql.pl
View file

@ -33,15 +33,6 @@ $dbh->do(qq{insert into
) )
}) or die $dbh->errstr; }) or die $dbh->errstr;
$dbh->do(qq{create table
config
(
attribute varchar(255) not null,
value varchar(255) not null,
primary key (attribute)
)
}) or die $dbh->errstr;
$dbh->do(qq{create table $dbh->do(qq{create table
users users
( (
@ -84,7 +75,12 @@ $dbh->do(qq{create table
coverage varchar(255) not null, coverage varchar(255) not null,
rights varchar(255) not null, rights varchar(255) not null,
license varchar(255) not null, license varchar(255) not null,
status int default 0, filesize int not null,
duration int not null,
width smallint not null,
height smallint not null,
fps float not null,
hash char(64) not null,
primary key (id) primary key (id)
) )
}) or die $dbh->errstr; }) or die $dbh->errstr;

4
trunk/locale/en-us.xml
View file

@ -65,6 +65,10 @@
<string id="error_openid_bogus_url">Invalid URL</string> <string id="error_openid_bogus_url">Invalid URL</string>
<string id="error_openid_no_head_tag">URL provided doesn't seem to have a head tag</string> <string id="error_openid_no_head_tag">URL provided doesn't seem to have a head tag</string>
<string id="error_openid_url_fetch_error">Error fetching the provided URL</string> <string id="error_openid_url_fetch_error">Error fetching the provided URL</string>
<string id="error_upload_invalid_stream">Invalid audio and/or video stream</string>
<string id="error_upload_io">Cannot read video file</string>
<string id="error_upload_not_a_video">File is not a video</string>
<string id="error_upload_duplicate">Video has already been uploaded</string>
<string id=""></string> <string id=""></string>
<!-- information --> <!-- information -->

143
trunk/tools/daemon.pl
View file

@ -39,142 +39,22 @@ sub interrupt
$dbh = DBI->connect("DBI:mysql:$database:$dbhost", $dbuser, $dbpass) or interrupt "could not connect to db"; $dbh = DBI->connect("DBI:mysql:$database:$dbhost", $dbuser, $dbpass) or interrupt "could not connect to db";
#video status:
# 0 - new entry - nothing done yet
# 1 - valid public video
# 2 - error: invalid audio and/or video stream
# 3 - error: file not found
# 4 - error: file is not a video
# 5 - error: video is a duplicate
while(1) while(1)
{ {
#get fresh video id from db #get fresh video id from db
my $sth = $dbh->prepare(qq{select id from uploaded where status = 0 limit 1}) or interrupt $dbh->errstr; my $sth = $dbh->prepare(qq{select id, filesize, duration, width, height, fps, hash
from uploaded where duration != 0 and width != 0 and height != 0 limit 1}) or interrupt $dbh->errstr;
$sth->execute() or interrupt $dbh->errstr; $sth->execute() or interrupt $dbh->errstr;
my ($id) = $sth->fetchrow_array(); my ($id,$filesize, $duration, $width, $height, $fps, $sha) = $sth->fetchrow_array();
$sth->finish() or interrupt $dbh->errstr; $sth->finish() or interrupt $dbh->errstr;
if($id) if($id)
{ {
$info = `export SDL_VIDEODRIVER="dummy"; ffplay -stats -an -vn -nodisp /tmp/$id 2>&1`;
if($info =~ /ignoring/)
{
appendlog "id: $id",
"error: invalid stream",
"ffplay msg: $info";
#write status 2 to uploaded table
$dbh->do(qq{update uploaded set status = ? where id = ?}, undef, 2, $id) or interrupt $dbh->errstr;
unlink "/tmp/$id";
}
elsif ($info =~ /I\/O error occured/)
{
appendlog "id: $id",
"error: file not found",
"ffplay msg: $info";
#write status 3 to uploaded table
$dbh->do(qq{update uploaded set status = ? where id = ?}, undef, 3, $id) or interrupt $dbh->errstr;
unlink "/tmp/$id";
}
elsif ($info =~ /Unknown format/ or $info =~ /could not find codec parameters/)
{
appendlog "id: $id",
"error: file is of unknown format",
"ffplay msg: $info";
#write status 4 to uploaded table
$dbh->do(qq{update uploaded set status = ? where id = ?}, undef, 4, $id) or interrupt $dbh->errstr;
unlink "/tmp/$id";
}
else
{
$sha = new Digest::SHA(256);
$sha->addfile("/tmp/$id");
$sha = $sha->hexdigest;
#check if this hash is already in database
my $sth = $dbh->prepare(qq{select id from videos where hash = ? limit 1}) or interrupt $dbh->errstr;
$sth->execute($sha) or interrupt $dbh->errstr;
my ($resultid) = $sth->fetchrow_array();
$sth->finish() or interrupt $dbh->errstr;
#if so, then video is a duplicate (alternatively ALL HAIL QUANTUM COMPUTING)
if($resultid)
{
appendlog "id: $id",
"error: video already uploaded: $resultid";
#write status 5 to uploaded table
$dbh->do(qq{update uploaded set status = ? where id = ?}, undef, 5, $id) or interrupt $dbh->errstr;
unlink "/tmp/$id";
}
else
{
($container, $duration) = $info =~ /Input \#0, (\w+),.+?\n.+?Duration: (\d{2}:\d{2}:\d{2}\.\d)/;
#these two regexes have to be applied seperately because nobody knows which stream (audio or video) comes first
($audio) = $info =~ /Audio: (\w+)/;
($video, $width, $height, $fps) = $info =~ /Video: ([\w\d]+),.+?(\d+)x(\d+),.+?(\d+\.\d+) fps/;
if(!$audio or !$video or !$duration)
{
appendlog "id: $id",
"error: error: stream is missing or video is corrupt",
"audio: $audio",
"video: $video",
"duration: $duration",
"ffplay msg: $info";
#write status 2 to uploaded table
$dbh->do(qq{update uploaded set status = ? where id = ?}, undef, 2, $id) or interrupt $dbh->errstr;
unlink "/tmp/$id";
}
else
{
$filesize = -s "/tmp/$id";
#convert hh:mm:ss.s duration to full seconds - thanks perl for making this so damn easy!
#don't want to know how this would look in python or php... hell I don't even have to create extra variables!
$duration =~ /^(\d{2}):(\d{2}):(\d{2})\.(\d)$/;
$duration = int($1*3600 + $2*60 + $3 + $4/10 + .5);
#create thumbnail
$thumbnailsec = int(rand($duration));
$previewsec = $thumbnailsec;
#the width/height calculation could of course be much shorter but less readable then
#all thumbs have equal height
$tnmaxheight = 120;
$tnheight = $tnmaxheight;
$tnwidth = int($tnheight*($width/$height)/2 + .5)*2;
system "ffmpeg -i /tmp/$id -vcodec mjpeg -vframes 1 -an -f rawvideo -ss $thumbnailsec -s ".$tnwidth."x$tnheight $root/video-stills/thumbnails/$id";
system "ffmpeg -i /tmp/$id -vcodec mjpeg -vframes 1 -an -f rawvideo -ss $previewsec $root/video-stills/previews/$id";
$vmaxheight = 640; $vmaxheight = 640;
#check if the upload already is in the right format and smaller/equal max-width/height #video height is either the maximum video height
if ($container eq 'ogg' and $video eq 'theora' and $audio eq 'vorbis' and $height <= $vmaxheight) #or (when the original is smaller than that) the original height
{
appendlog $id, "file already is ogg-theora/vorbis";
#add video to videos table
$dbh->do(qq{insert into videos select id, title, description, userid, timestamp, creator,
subject, source, language, coverage, rights, license, ?, ?, ?, ?, ?, ?, 0, 0
from uploaded where id = ?}, undef, $filesize, $duration, $width,
$height, $fps, $sha, $id) or interrupt $dbh->errstr;
#move video
move "/tmp/$id", "$root/videos/$id";
}
else #encode video
{
#video height is either ther maximum video height
#or when the original is smaller than that the original height
#check for multiple by 8 #check for multiple by 8
$vheight = $vmaxheight <= $height ? $vmaxheight : int($height/8 + .5)*8; $vheight = $vmaxheight <= $height ? $vmaxheight : int($height/8 + .5)*8;
$vwidth = int($vheight*($width/$height)/8 + .5)*8; $vwidth = int($vheight*($width/$height)/8 + .5)*8;
@ -191,23 +71,18 @@ while(1)
#add video to videos table #add video to videos table
$dbh->do(qq{insert into videos select id, title, description, userid, timestamp, creator, $dbh->do(qq{insert into videos select id, title, description, userid, timestamp, creator,
subject, source, language, coverage, rights, license, ?, ?, ?, ?, ?, ?, 0, 0 subject, source, language, coverage, rights, license, ?, duration, ?, ?, fps, hash, 0, 0
from uploaded where id = ?}, undef, $filesize, $duration, $vwidth, from uploaded where id = ?}, undef, $filesize, $vwidth,
$vheight, $fps, $sha, $id) or interrupt $dbh->errstr; $vheight, $id) or interrupt $dbh->errstr;
#delete temp file #delete temp file
unlink "/tmp/$id"; unlink "/tmp/$id";
}
#create torrent file
#TODO:create torrent file
#delete from uploaded table #delete from uploaded table
$dbh->do(qq{delete from uploaded where id = ?}, undef, $id) or interrupt $dbh->errstr; $dbh->do(qq{delete from uploaded where id = ?}, undef, $id) or interrupt $dbh->errstr;
} }
}
}
}
else else
{ {
#TODO: maybe make this event-driven by using the kernels has-this-file-changed-interface ? #TODO: maybe make this event-driven by using the kernels has-this-file-changed-interface ?

110
trunk/uploader.pl
View file

@ -48,7 +48,117 @@ if($userinfo->{'id'} && $query->param("DC.Title") &&
} }
close TEMPFILE; close TEMPFILE;
$info = `export SDL_VIDEODRIVER="dummy"; ffplay -stats -an -vn -nodisp /tmp/$id 2>&1`;
if($info =~ /ignoring/)
{
#delete from uploaded table
$dbh->do(qq{delete from uploaded where id = ?}, undef, $id) or die $dbh->errstr;
unlink "/tmp/$id";
print $query->redirect("index.pl?error=error_upload_invalid_stream");
}
elsif ($info =~ /I\/O error occured/)
{
#delete from uploaded table
$dbh->do(qq{delete from uploaded where id = ?}, undef, $id) or die $dbh->errstr;
unlink "/tmp/$id";
print $query->redirect("index.pl?error=error_upload_io");
}
elsif ($info =~ /Unknown format/ or $info =~ /could not find codec parameters/)
{
#delete from uploaded table
$dbh->do(qq{delete from uploaded where id = ?}, undef, $id) or die $dbh->errstr;
unlink "/tmp/$id";
print $query->redirect("index.pl?error=error_upload_not_a_video");
}
else
{
$sha = new Digest::SHA(256);
$sha->addfile("/tmp/$id");
$sha = $sha->hexdigest;
#check if this hash is already in database
my $sth = $dbh->prepare(qq{select id from videos where hash = ? limit 1}) or die $dbh->errstr;
$sth->execute($sha) or die $dbh->errstr;
my ($resultid) = $sth->fetchrow_array();
$sth->finish() or die $dbh->errstr;
#if so, then video is a duplicate (alternatively ALL HAIL QUANTUM COMPUTING)
if($resultid)
{
#delete from uploaded table
$dbh->do(qq{delete from uploaded where id = ?}, undef, $id) or die $dbh->errstr;
unlink "/tmp/$id";
print $query->redirect("index.pl?error=error_upload_duplicate");
}
else
{
($container, $duration) = $info =~ /Input \#0, (\w+),.+?\n.+?Duration: (\d{2}:\d{2}:\d{2}\.\d)/;
#these two regexes have to be applied seperately because nobody knows which stream (audio or video) comes first
($audio) = $info =~ /Audio: (\w+)/;
($video, $width, $height, $fps) = $info =~ /Video: ([\w\d]+),.+?(\d+)x(\d+),.+?(\d+\.\d+) fps/;
if(!$video or !$duration)
{
#delete from uploaded table
$dbh->do(qq{delete from uploaded where id = ?}, undef, $id) or die $dbh->errstr;
unlink "/tmp/$id";
print $query->redirect("index.pl?error=error_upload_invalid_stream");
}
else
{
$filesize = -s "/tmp/$id";
#convert hh:mm:ss.s duration to full seconds - thanks perl for making this so damn easy!
#don't want to know how this would look in python or php... hell I don't even have to create extra variables!
$duration =~ /^(\d{2}):(\d{2}):(\d{2})\.(\d)$/;
$duration = int($1*3600 + $2*60 + $3 + $4/10 + .5);
#create thumbnail
$thumbnailsec = int(rand($duration));
$previewsec = $thumbnailsec;
#the width/height calculation could of course be much shorter but less readable then
#all thumbs have equal height
$tnmaxheight = 120;
$tnheight = $tnmaxheight;
$tnwidth = int($tnheight*($width/$height)/2 + .5)*2;
system "ffmpeg -i /tmp/$id -vcodec mjpeg -vframes 1 -an -f rawvideo -ss $thumbnailsec -s ".$tnwidth."x$tnheight $root/video-stills/thumbnails/$id";
system "ffmpeg -i /tmp/$id -vcodec mjpeg -vframes 1 -an -f rawvideo -ss $previewsec $root/video-stills/previews/$id";
$vmaxheight = 640;
#check if the upload already is in the right format and smaller/equal max-width/height
if ($container eq 'ogg' and $video eq 'theora' and ($audio eq 'vorbis' or not $audio) and $height <= $vmaxheight)
{
#add video to videos table
$dbh->do(qq{insert into videos select id, title, description, userid, timestamp, creator,
subject, source, language, coverage, rights, license, ?, ?, ?, ?, ?, ?, 0, 0
from uploaded where id = ?}, undef, $filesize, $duration, $width,
$height, $fps, $sha, $id) or die $dbh->errstr;
#delete from uploaded table
$dbh->do(qq{delete from uploaded where id = ?}, undef, $id) or die $dbh->errstr;
move("/tmp/$id", "$root/videos/$id");
#TODO:create torrent file
}
else
{
#write all valueable information to database so the daemon can fetch it
$dbh->do(qq{update uploaded set filesize = ?, duration = ?, width = ?,
height = ?, fps = ?, hash = ? where id = ?}, undef, $filesize, $duration, $width,
$height, $fps, $sha, $id) or die $dbh->errstr;
}
#print success to the user
print $query->redirect("index.pl?information=information_uploaded&value=$domain/video/".urlencode($query->param("DC.Title"))."/$id/"); print $query->redirect("index.pl?information=information_uploaded&value=$domain/video/".urlencode($query->param("DC.Title"))."/$id/");
}
}
}
} }
else else
{ {