josch
/
yolanda
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

restructured upload process

Browse Source 
git-svn-id: http://yolanda.mister-muffin.de/svn@286 7eef14d0-6ed0-489d-bf55-20463b2d70db
main
josch 16 years ago
parent a5e3e3f198
commit 9d7fea9515
5 changed files with 168 additions and 182 deletions
Show Stats Download Patch File Download Diff File

1
trunk/include.pl
Unescape Escape View File

@ -8,6 +8,7 @@ use Digest::SHA qw(sha256_hex);
use LWPx::ParanoidAgent;
use Net::OpenID::Consumer;
use CGI::Carp qw(fatalsToBrowser set_message);
use File::Copy;
set_message("It's not a bug, it's a feature!!<br />(include this error message in your bugreport here: <a href=\"http://yolanda.mister-muffin.de/newticket\">Yolanda bugtracker</a>)");

42
trunk/init_sql.pl
Unescape Escape View File

@ -33,15 +33,6 @@ $dbh->do(qq{insert into
)
}) or die $dbh->errstr;
$dbh->do(qq{create table
config
(
attribute varchar(255) not null,
value varchar(255) not null,
primary key (attribute)
)
}) or die $dbh->errstr;
$dbh->do(qq{create table
users
(
@ -72,20 +63,25 @@ $dbh->do(qq{insert into
$dbh->do(qq{create table
uploaded
(
id int auto_increment not null,
title varchar(255) not null,
description text not null,
userid int not null,
timestamp bigint not null,
creator varchar(255) not null,
subject varchar(255) not null,
source varchar(255) not null,
language varchar(255) not null,
coverage varchar(255) not null,
rights varchar(255) not null,
license varchar(255) not null,
status int default 0,
primary key (id)
id int auto_increment not null,
title varchar(255) not null,
description text not null,
userid int not null,
timestamp bigint not null,
creator varchar(255) not null,
subject varchar(255) not null,
source varchar(255) not null,
language varchar(255) not null,
coverage varchar(255) not null,
rights varchar(255) not null,
license varchar(255) not null,
filesize int not null,
duration int not null,
width smallint not null,
height smallint not null,
fps float not null,
hash char(64) not null,
primary key (id)
)
}) or die $dbh->errstr;

4
trunk/locale/en-us.xml
Unescape Escape View File

@ -65,6 +65,10 @@
<string id="error_openid_bogus_url">Invalid URL</string>
<string id="error_openid_no_head_tag">URL provided doesn't seem to have a head tag</string>
<string id="error_openid_url_fetch_error">Error fetching the provided URL</string>
<string id="error_upload_invalid_stream">Invalid audio and/or video stream</string>
<string id="error_upload_io">Cannot read video file</string>
<string id="error_upload_not_a_video">File is not a video</string>
<string id="error_upload_duplicate">Video has already been uploaded</string>
<string id=""></string>
<!-- information -->

191
trunk/tools/daemon.pl
Unescape Escape View File

@ -39,174 +39,49 @@ sub interrupt
$dbh = DBI->connect("DBI:mysql:$database:$dbhost", $dbuser, $dbpass) or interrupt "could not connect to db";
#video status:
# 0 - new entry - nothing done yet
# 1 - valid public video
# 2 - error: invalid audio and/or video stream
# 3 - error: file not found
# 4 - error: file is not a video
# 5 - error: video is a duplicate
while(1)
{
#get fresh video id from db
my $sth = $dbh->prepare(qq{select id from uploaded where status = 0 limit 1}) or interrupt $dbh->errstr;
my $sth = $dbh->prepare(qq{select id, filesize, duration, width, height, fps, hash
from uploaded where duration != 0 and width != 0 and height != 0 limit 1}) or interrupt $dbh->errstr;
$sth->execute() or interrupt $dbh->errstr;
my ($id) = $sth->fetchrow_array();
my ($id,$filesize, $duration, $width, $height, $fps, $sha) = $sth->fetchrow_array();
$sth->finish() or interrupt $dbh->errstr;
if($id)
{
$info = `export SDL_VIDEODRIVER="dummy"; ffplay -stats -an -vn -nodisp /tmp/$id 2>&1`;
$vmaxheight = 640;
#video height is either the maximum video height
#or (when the original is smaller than that) the original height
#check for multiple by 8
$vheight = $vmaxheight <= $height ? $vmaxheight : int($height/8 + .5)*8;
$vwidth = int($vheight*($width/$height)/8 + .5)*8;
$abitrate = 64;
$vbitrate = int(($filesize*8) / $duration + .5) - $abitrate;
#TODO: add metadata information
system "ffmpeg2theora --optimize --videobitrate $vbitrate --audiobitrate $abitrate --sharpness 0 --width $vwidth --height $vheight --output $root/videos/$id /tmp/$id";
appendlog $id, $audio, $video, $vwidth, $vheight, $fps, $duration, $sha;
$filesize = -s "$root/videos/$id";
#add video to videos table
$dbh->do(qq{insert into videos select id, title, description, userid, timestamp, creator,
subject, source, language, coverage, rights, license, ?, duration, ?, ?, fps, hash, 0, 0
from uploaded where id = ?}, undef, $filesize, $vwidth,
$vheight, $id) or interrupt $dbh->errstr;
#delete temp file
unlink "/tmp/$id";
#TODO:create torrent file
if($info =~ /ignoring/)
{
appendlog "id: $id",
"error: invalid stream",
"ffplay msg: $info";
#write status 2 to uploaded table
$dbh->do(qq{update uploaded set status = ? where id = ?}, undef, 2, $id) or interrupt $dbh->errstr;
unlink "/tmp/$id";
}
elsif ($info =~ /I\/O error occured/)
{
appendlog "id: $id",
"error: file not found",
"ffplay msg: $info";
#write status 3 to uploaded table
$dbh->do(qq{update uploaded set status = ? where id = ?}, undef, 3, $id) or interrupt $dbh->errstr;
unlink "/tmp/$id";
}
elsif ($info =~ /Unknown format/ or $info =~ /could not find codec parameters/)
{
appendlog "id: $id",
"error: file is of unknown format",
"ffplay msg: $info";
#write status 4 to uploaded table
$dbh->do(qq{update uploaded set status = ? where id = ?}, undef, 4, $id) or interrupt $dbh->errstr;
unlink "/tmp/$id";
}
else
{
$sha = new Digest::SHA(256);
$sha->addfile("/tmp/$id");
$sha = $sha->hexdigest;
#check if this hash is already in database
my $sth = $dbh->prepare(qq{select id from videos where hash = ? limit 1}) or interrupt $dbh->errstr;
$sth->execute($sha) or interrupt $dbh->errstr;
my ($resultid) = $sth->fetchrow_array();
$sth->finish() or interrupt $dbh->errstr;
#if so, then video is a duplicate (alternatively ALL HAIL QUANTUM COMPUTING)
if($resultid)
{
appendlog "id: $id",
"error: video already uploaded: $resultid";
#write status 5 to uploaded table
$dbh->do(qq{update uploaded set status = ? where id = ?}, undef, 5, $id) or interrupt $dbh->errstr;
unlink "/tmp/$id";
}
else
{
($container, $duration) = $info =~ /Input \#0, (\w+),.+?\n.+?Duration: (\d{2}:\d{2}:\d{2}\.\d)/;
#these two regexes have to be applied seperately because nobody knows which stream (audio or video) comes first
($audio) = $info =~ /Audio: (\w+)/;
($video, $width, $height, $fps) = $info =~ /Video: ([\w\d]+),.+?(\d+)x(\d+),.+?(\d+\.\d+) fps/;
if(!$audio or !$video or !$duration)
{
appendlog "id: $id",
"error: error: stream is missing or video is corrupt",
"audio: $audio",
"video: $video",
"duration: $duration",
"ffplay msg: $info";
#write status 2 to uploaded table
$dbh->do(qq{update uploaded set status = ? where id = ?}, undef, 2, $id) or interrupt $dbh->errstr;
unlink "/tmp/$id";
}
else
{
$filesize = -s "/tmp/$id";
#convert hh:mm:ss.s duration to full seconds - thanks perl for making this so damn easy!
#don't want to know how this would look in python or php... hell I don't even have to create extra variables!
$duration =~ /^(\d{2}):(\d{2}):(\d{2})\.(\d)$/;
$duration = int($1*3600 + $2*60 + $3 + $4/10 + .5);
#create thumbnail
$thumbnailsec = int(rand($duration));
$previewsec = $thumbnailsec;
#the width/height calculation could of course be much shorter but less readable then
#all thumbs have equal height
$tnmaxheight = 120;
$tnheight = $tnmaxheight;
$tnwidth = int($tnheight*($width/$height)/2 + .5)*2;
system "ffmpeg -i /tmp/$id -vcodec mjpeg -vframes 1 -an -f rawvideo -ss $thumbnailsec -s ".$tnwidth."x$tnheight $root/video-stills/thumbnails/$id";
system "ffmpeg -i /tmp/$id -vcodec mjpeg -vframes 1 -an -f rawvideo -ss $previewsec $root/video-stills/previews/$id";
$vmaxheight = 640;
#check if the upload already is in the right format and smaller/equal max-width/height
if ($container eq 'ogg' and $video eq 'theora' and $audio eq 'vorbis' and $height <= $vmaxheight)
{
appendlog $id, "file already is ogg-theora/vorbis";
#add video to videos table
$dbh->do(qq{insert into videos select id, title, description, userid, timestamp, creator,
subject, source, language, coverage, rights, license, ?, ?, ?, ?, ?, ?, 0, 0
from uploaded where id = ?}, undef, $filesize, $duration, $width,
$height, $fps, $sha, $id) or interrupt $dbh->errstr;
#move video
move "/tmp/$id", "$root/videos/$id";
}
else #encode video
{
#video height is either ther maximum video height
#or when the original is smaller than that the original height
#check for multiple by 8
$vheight = $vmaxheight <= $height ? $vmaxheight : int($height/8 + .5)*8;
$vwidth = int($vheight*($width/$height)/8 + .5)*8;
$abitrate = 64;
$vbitrate = int(($filesize*8) / $duration + .5) - $abitrate;
#TODO: add metadata information
system "ffmpeg2theora --optimize --videobitrate $vbitrate --audiobitrate $abitrate --sharpness 0 --width $vwidth --height $vheight --output $root/videos/$id /tmp/$id";
appendlog $id, $audio, $video, $vwidth, $vheight, $fps, $duration, $sha;
$filesize = -s "$root/videos/$id";
#add video to videos table
$dbh->do(qq{insert into videos select id, title, description, userid, timestamp, creator,
subject, source, language, coverage, rights, license, ?, ?, ?, ?, ?, ?, 0, 0
from uploaded where id = ?}, undef, $filesize, $duration, $vwidth,
$vheight, $fps, $sha, $id) or interrupt $dbh->errstr;
#delete temp file
unlink "/tmp/$id";
}
#create torrent file
#delete from uploaded table
$dbh->do(qq{delete from uploaded where id = ?}, undef, $id) or interrupt $dbh->errstr;
}
}
}
#delete from uploaded table
$dbh->do(qq{delete from uploaded where id = ?}, undef, $id) or interrupt $dbh->errstr;
}
else
{

112
trunk/uploader.pl
Unescape Escape View File

@ -48,7 +48,117 @@ if($userinfo->{'id'} && $query->param("DC.Title") &&
}
close TEMPFILE;
print $query->redirect("index.pl?information=information_uploaded&value=$domain/video/".urlencode($query->param("DC.Title"))."/$id/");
$info = `export SDL_VIDEODRIVER="dummy"; ffplay -stats -an -vn -nodisp /tmp/$id 2>&1`;
if($info =~ /ignoring/)
{
#delete from uploaded table
$dbh->do(qq{delete from uploaded where id = ?}, undef, $id) or die $dbh->errstr;
unlink "/tmp/$id";
print $query->redirect("index.pl?error=error_upload_invalid_stream");
}
elsif ($info =~ /I\/O error occured/)
{
#delete from uploaded table
$dbh->do(qq{delete from uploaded where id = ?}, undef, $id) or die $dbh->errstr;
unlink "/tmp/$id";
print $query->redirect("index.pl?error=error_upload_io");
}
elsif ($info =~ /Unknown format/ or $info =~ /could not find codec parameters/)
{
#delete from uploaded table
$dbh->do(qq{delete from uploaded where id = ?}, undef, $id) or die $dbh->errstr;
unlink "/tmp/$id";
print $query->redirect("index.pl?error=error_upload_not_a_video");
}
else
{
$sha = new Digest::SHA(256);
$sha->addfile("/tmp/$id");
$sha = $sha->hexdigest;
#check if this hash is already in database
my $sth = $dbh->prepare(qq{select id from videos where hash = ? limit 1}) or die $dbh->errstr;
$sth->execute($sha) or die $dbh->errstr;
my ($resultid) = $sth->fetchrow_array();
$sth->finish() or die $dbh->errstr;
#if so, then video is a duplicate (alternatively ALL HAIL QUANTUM COMPUTING)
if($resultid)
{
#delete from uploaded table
$dbh->do(qq{delete from uploaded where id = ?}, undef, $id) or die $dbh->errstr;
unlink "/tmp/$id";
print $query->redirect("index.pl?error=error_upload_duplicate");
}
else
{
($container, $duration) = $info =~ /Input \#0, (\w+),.+?\n.+?Duration: (\d{2}:\d{2}:\d{2}\.\d)/;
#these two regexes have to be applied seperately because nobody knows which stream (audio or video) comes first
($audio) = $info =~ /Audio: (\w+)/;
($video, $width, $height, $fps) = $info =~ /Video: ([\w\d]+),.+?(\d+)x(\d+),.+?(\d+\.\d+) fps/;
if(!$video or !$duration)
{
#delete from uploaded table
$dbh->do(qq{delete from uploaded where id = ?}, undef, $id) or die $dbh->errstr;
unlink "/tmp/$id";
print $query->redirect("index.pl?error=error_upload_invalid_stream");
}
else
{
$filesize = -s "/tmp/$id";
#convert hh:mm:ss.s duration to full seconds - thanks perl for making this so damn easy!
#don't want to know how this would look in python or php... hell I don't even have to create extra variables!
$duration =~ /^(\d{2}):(\d{2}):(\d{2})\.(\d)$/;
$duration = int($1*3600 + $2*60 + $3 + $4/10 + .5);
#create thumbnail
$thumbnailsec = int(rand($duration));
$previewsec = $thumbnailsec;
#the width/height calculation could of course be much shorter but less readable then
#all thumbs have equal height
$tnmaxheight = 120;
$tnheight = $tnmaxheight;
$tnwidth = int($tnheight*($width/$height)/2 + .5)*2;
system "ffmpeg -i /tmp/$id -vcodec mjpeg -vframes 1 -an -f rawvideo -ss $thumbnailsec -s ".$tnwidth."x$tnheight $root/video-stills/thumbnails/$id";
system "ffmpeg -i /tmp/$id -vcodec mjpeg -vframes 1 -an -f rawvideo -ss $previewsec $root/video-stills/previews/$id";
$vmaxheight = 640;
#check if the upload already is in the right format and smaller/equal max-width/height
if ($container eq 'ogg' and $video eq 'theora' and ($audio eq 'vorbis' or not $audio) and $height <= $vmaxheight)
{
#add video to videos table
$dbh->do(qq{insert into videos select id, title, description, userid, timestamp, creator,
subject, source, language, coverage, rights, license, ?, ?, ?, ?, ?, ?, 0, 0
from uploaded where id = ?}, undef, $filesize, $duration, $width,
$height, $fps, $sha, $id) or die $dbh->errstr;
#delete from uploaded table
$dbh->do(qq{delete from uploaded where id = ?}, undef, $id) or die $dbh->errstr;
move("/tmp/$id", "$root/videos/$id");
#TODO:create torrent file
}
else
{
#write all valueable information to database so the daemon can fetch it
$dbh->do(qq{update uploaded set filesize = ?, duration = ?, width = ?,
height = ?, fps = ?, hash = ? where id = ?}, undef, $filesize, $duration, $width,
$height, $fps, $sha, $id) or die $dbh->errstr;
}
#print success to the user
print $query->redirect("index.pl?information=information_uploaded&value=$domain/video/".urlencode($query->param("DC.Title"))."/$id/");
}
}
}
}
else
{

Write Preview
Loading…
Cancel
Save