Merge pull request #104 from karjonas/rtmpcheck
Adding a try_download function for RTMPDownloader
diff --git a/.gitignore b/.gitignore
index feeb605..6e27663 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,6 +9,7 @@
*.old
*.mp4
*.flv
+*.m3u8
*.ogg
*.webm
*.gz
diff --git a/debian/control b/debian/control
index 2f94fb9..e8e0865 100644
--- a/debian/control
+++ b/debian/control
@@ -15,8 +15,9 @@
Package: get-flash-videos
Architecture: all
-Depends: libcrypt-blowfish-perl,
+Depends: libcrypt-blowfish-pp-perl,
libdata-amf-perl,
+ libencode-locale-perl,
libhtml-parser-perl,
libhtml-tree-perl,
libmodule-find-perl,
@@ -24,14 +25,14 @@
liburi-perl,
libwww-mechanize-perl,
libwww-perl,
- rtmpdump,
${misc:Depends},
${perl:Depends}
-Recommends: get_iplayer,
+Recommends: get-iplayer,
ffmpeg | libav-tools,
- libcrypt-rijndeal-perl,
+ libcrypt-rijndael-perl,
liblwp-protocol-socks-perl,
- libxml-simple-perl
+ libxml-simple-perl,
+ rtmpdump|flvstreamer
Suggests: mplayer
Description: Video downloader for various Flash-based video hosting sites
Download videos from various Flash-based video hosting sites, without having
diff --git a/get_flash_videos b/get_flash_videos
index 8aaa6fc..6f70dcf 100755
--- a/get_flash_videos
+++ b/get_flash_videos
@@ -410,7 +410,7 @@
if(ref $data eq 'HASH') {
if (defined($data->{downloader}) && $data->{downloader} eq "ffmpeg") {
$downloader = FlashVideo::FFmpegDownloader->new;
- $file ||= $data->{file};
+ $file ||= $data->{flv};
} else {
# RTMP data
$downloader = FlashVideo::RTMPDownloader->new;
@@ -695,8 +695,11 @@
debug "Trying to open plugin $plugin_dir/$plugin_name";
- if (open my $plugin_fh, '<', "$plugin_dir/$plugin_name") {
- return $plugin_fh; # Perl then reads the plugin from the FH
+ if (-s "$plugin_dir/$plugin_name") {
+ if (open my $plugin_fh, '<', "$plugin_dir/$plugin_name") {
+ return $plugin_fh; # Perl then reads the plugin from the FH
+ }
+ info "Failed to open plugin $plugin_dir/$plugin_name $!";
}
}
diff --git a/lib/FlashVideo/Generic.pm b/lib/FlashVideo/Generic.pm
index a3b0e49..3f026a6 100644
--- a/lib/FlashVideo/Generic.pm
+++ b/lib/FlashVideo/Generic.pm
@@ -6,6 +6,10 @@
use URI;
use FlashVideo::URLFinder;
use URI::Escape qw(uri_unescape);
+use HTML::Entities qw(decode_entities);
+
+our $VERSION = '0.01';
+sub Version() { $VERSION; }
my $video_re = qr!http[-:/a-z0-9%_.?=&]+@{[EXTENSIONS]}
# Grab any params that might be used for auth..
@@ -70,7 +74,9 @@
if(!$actual_url) {
for my $iframe($browser->content =~ /<iframe[^>]+src=["']?([^"'>]+)/gi) {
+ $iframe = decode_entities($iframe);
$iframe = URI->new_abs($iframe, $browser->uri);
+ $iframe = decode_entities($iframe);
debug "Found iframe: $iframe";
my $sub_browser = $browser->clone;
$sub_browser->get($iframe);
diff --git a/lib/FlashVideo/Site/Adultswim.pm b/lib/FlashVideo/Site/Adultswim.pm
index fca786e..1ad2943 100644
--- a/lib/FlashVideo/Site/Adultswim.pm
+++ b/lib/FlashVideo/Site/Adultswim.pm
@@ -4,6 +4,9 @@
use strict;
use FlashVideo::Utils;
+our $VERSION = '0.02';
+sub Version() { $VERSION; }
+
sub find_video {
my($self, $browser, $embed_url) = @_;
@@ -18,10 +21,8 @@
}
}
- my $id1;
- if($segIds =~ m/^([^#]*)#/){
- $id1 = $1;
- }
+ ($segIds)=$browser->{content} =~ m/<section[^>]* ?data-segment-ids=["'](.+?)["'] ?[^>]*>/ if(!$segIds);
+ my ($id1) = $segIds =~ m/^([0-9a-f]+)/;
my $title;
if($browser->{content} =~ m/<meta property=["']og:title["'] content=["']([^"']+)["']\/>/){
@@ -66,22 +67,16 @@
$browser->get($videoURL);
$xml = from_xml($browser);
-
-# my $pick;
-# foreach(@{$xml->{entry}}){
-# if(!($_->{ref}->{href} =~ m/iPhone/)){
-# $pick = $1;
-# }
-# }
-
-# grep { $_->{name} eq "mimeType" } @{$_->{param}})[0]->{value}
-# my $pick = (grep { $_->{param}->{value}->[3] eq "video/x-flv" } @{$xml->{entry} } )[0];
-
- my $pick = $xml->{entry}[4];
-
- my $file_url = $pick->{ref}->{href};
-
- # $prefs->{quality}
+ my $bitrate=-1;
+ my $file_url;
+ foreach(@{$xml->{entry}}){
+ next if(ref($_) ne 'HASH');
+ next if ($_->{ref}->{href} =~ m,\.akamaihd\.net\/,);
+ next if ($_->{param}->{bitrate} < $bitrate && $_->{ref}->{href} =~ m/iPhone/);
+ $file_url=$_->{ref}->{href};
+ $bitrate=$_->{param}->{bitrate};
+ #print STDERR $_->{param}->{bitrate}."\t".$_->{ref}->{href}."\n";
+ }
return $file_url, title_to_filename($title);
}
diff --git a/lib/FlashVideo/Site/Channel5.pm b/lib/FlashVideo/Site/Channel5.pm
index 643b3f6..f55acf8 100644
--- a/lib/FlashVideo/Site/Channel5.pm
+++ b/lib/FlashVideo/Site/Channel5.pm
@@ -5,6 +5,9 @@
use FlashVideo::Utils;
use MIME::Base64;
+our $VERSION = '0.01';
+sub Version() { $VERSION; }
+
my $encode_rates = {
"low" => 480,
"medium" => 800,
@@ -226,9 +229,6 @@
my($self, $browser, $url) = @_;
return 1 if $url && URI->new($url)->host =~ /\.channel5\.com$/;
-
- return $browser->content =~ /(playerI[dD]|brightcove.player.create)/
- && $browser->content =~ /brightcove/i;
}
1;
diff --git a/lib/FlashVideo/Site/Cultureunplugged.pm b/lib/FlashVideo/Site/Cultureunplugged.pm
new file mode 100644
index 0000000..2803dd7
--- /dev/null
+++ b/lib/FlashVideo/Site/Cultureunplugged.pm
@@ -0,0 +1,24 @@
+# Part of get-flash-videos. See get_flash_videos for copyright.
+package FlashVideo::Site::Cultureunplugged;
+
+use strict;
+use FlashVideo::JSON;
+use FlashVideo::Utils;
+use URI::Escape;
+
+our $VERSION = '0.01';
+sub Version() { $VERSION; }
+
+sub find_video {
+ my ($self, $browser, $embed_url) = @_;
+
+ my ($id, $title) = $embed_url =~ m{/play/(\d+)/(.*?)};
+
+ die "No video ID found" unless $id;
+
+ $browser->get("http://www.cultureunplugged.com/ajax/getMovieInfo.php?movieId=$id&type=");
+ my ($json) = from_json($browser->content);
+ return $json->{'url'}, title_to_filename($json->{'title'}, "mp4");
+}
+
+1;
diff --git a/lib/FlashVideo/Site/Facebook.pm b/lib/FlashVideo/Site/Facebook.pm
new file mode 100644
index 0000000..ba2aff1
--- /dev/null
+++ b/lib/FlashVideo/Site/Facebook.pm
@@ -0,0 +1,32 @@
+# Part of get-flash-videos. See get_flash_videos for copyright.
+package FlashVideo::Site::Facebook;
+
+use strict;
+use FlashVideo::Utils;
+
+use URI::Escape;
+
+our $VERSION = '0.01';
+sub Version { $VERSION; }
+
+sub find_video {
+ my ($self, $browser, $embed_url) = @_;
+
+ # If we should process Facebook's Like button, leave
+ return if ($embed_url =~ /http:\/\/www\.facebook\.com\/plugins\/like\.php/);
+
+ # Grab the file from the page..
+ my $params = ($browser->content =~ /\["params","(.+?)"\]/)[0];
+ $params =~ s/\\u([[:xdigit:]]{1,4})/chr(eval("0x$1"))/egis;
+ $params = uri_unescape($params);
+ my $url = ($params =~ /"hd_src":"([^"]*)"/)[0];
+ if (!$url) { $url = ($params =~ /"sd_src":"([^"]*)"/)[0]; }
+ $url =~ s/\\\//\//g;
+ die "Unable to extract url" unless $url;
+
+ my $filename = ($url =~ /([^\/]*)\?/)[0];
+
+ return $url, $filename;
+}
+
+1;
diff --git a/lib/FlashVideo/Site/Gosupark.pm b/lib/FlashVideo/Site/Gosupark.pm
new file mode 100644
index 0000000..3e285a4
--- /dev/null
+++ b/lib/FlashVideo/Site/Gosupark.pm
@@ -0,0 +1,24 @@
+# Part of get-flash-videos. See get_flash_videos for copyright.
+package FlashVideo::Site::Gosupark;
+
+use strict;
+use FlashVideo::Utils;
+
+our $VERSION = '0.01';
+sub Version() { $VERSION; }
+
+sub find_video {
+ my ($self, $browser, $embed_url) = @_;
+ my $url = "";
+
+ if ($browser->content =~ /.*\s*file: "(http:\/\/gosupark[^"]+).*",/) {
+ $url = $1;
+ } else {
+ return;
+ }
+ debug ("URL: '" . $url . "'");
+ return $url, title_to_filename("", "mp4");
+}
+
+1;
+
diff --git a/lib/FlashVideo/Site/Pbs.pm b/lib/FlashVideo/Site/Pbs.pm
index c8ef937..b94d065 100644
--- a/lib/FlashVideo/Site/Pbs.pm
+++ b/lib/FlashVideo/Site/Pbs.pm
@@ -32,9 +32,12 @@
die "Must have Crypt::Rijndael installed to download from PBS"
unless eval { require Crypt::Rijndael };
- my ($media_id) = $browser->uri->as_string =~ m[
- ^http://video\.pbs\.org/video/(\d+)
- ]x;
+ my ($media_id) = $embed_url =~ m[http://video\.pbs\.org/videoPlayerInfo/(\d+)]x;
+ unless (defined $media_id) {
+ ($media_id) = $browser->uri->as_string =~ m[
+ ^http://video\.pbs\.org/video/(\d+)
+ ]x;
+ }
unless (defined $media_id) {
($media_id) = $browser->content =~ m[
http://video\.pbs\.org/widget/partnerplayer/(\d+)
@@ -49,6 +52,9 @@
($media_id) = $browser->content =~ m[var videoUrl = "([^"]+)"];
}
unless (defined $media_id) {
+ ($media_id) = $browser->content =~ m[pbs_video_id_\S+" value="([^"]+)"];
+ }
+ unless (defined $media_id) {
my ($pap_id, $youtube_id) = $browser->content =~ m[
\bDetectFlashDecision\ \('([^']+)',\ '([^']+)'\);
]x;
diff --git a/lib/FlashVideo/Site/Svtplay.pm b/lib/FlashVideo/Site/Svtplay.pm
index e588a95..9a52156 100644
--- a/lib/FlashVideo/Site/Svtplay.pm
+++ b/lib/FlashVideo/Site/Svtplay.pm
@@ -108,7 +108,7 @@
return {
downloader => "ffmpeg",
- file => $filename,
+ flv => $filename,
args => \@ffmpeg_args
};
} else {
diff --git a/lib/FlashVideo/Site/Video44.pm b/lib/FlashVideo/Site/Video44.pm
new file mode 100644
index 0000000..f3dbec9
--- /dev/null
+++ b/lib/FlashVideo/Site/Video44.pm
@@ -0,0 +1,49 @@
+# Part of get-flash-videos. See get_flash_videos for copyright.
+package FlashVideo::Site::Video44;
+
+use strict;
+use FlashVideo::Utils;
+use URI::Escape;
+
+our $VERSION = '0.01';
+sub Version() { $VERSION; }
+
+sub find_video {
+ my ($self, $browser, $embed_url) = @_;
+
+ my $flashvars = "";
+ my $file = "";
+ my $url = "";
+ my $name = "";
+
+ # get configuration passed to flash player
+ if ($browser->content =~ /\s*<param name="flashvars"\s*value="([^"]+)" \/>/) {
+ $flashvars = $1;
+ } else {
+ # if we can't get it, just leave as the video URL is there
+ debug("Can't find flashvars");
+ return;
+ }
+
+ debug ("Flashvars: " . $flashvars);
+
+ # in the configuration there is also URL we're looking for
+ if ($flashvars =~ /&file=(http[^&]+)&/) {
+ $file = $1;
+ } else {
+ debug("Can't find file");
+ return;
+ }
+
+ debug("File: " . $file);
+
+ $url = uri_unescape($file);
+ debug("URL: '" . $url . "'");
+
+ # URL ends with filename
+ $name = $url;
+ $name =~ s/.*\/([^\/]+)/$1/;
+ return $url, title_to_filename($name);
+}
+
+1;
diff --git a/lib/FlashVideo/Site/Videofun.pm b/lib/FlashVideo/Site/Videofun.pm
new file mode 100644
index 0000000..0d16d03
--- /dev/null
+++ b/lib/FlashVideo/Site/Videofun.pm
@@ -0,0 +1,39 @@
+# Part of get-flash-videos. See get_flash_videos for copyright.
+package FlashVideo::Site::Videofun;
+
+use strict;
+use FlashVideo::Utils;
+use URI::Escape;
+
+our $VERSION = '0.01';
+sub Version() { $VERSION; }
+
+sub find_video {
+ my ($self, $browser, $embed_url) = @_;
+
+ my $coded_url = "";
+ my $url = "";
+ my $name = "";
+
+
+ # read URL from the configuration passed to flash player
+ if ($browser->content =~ /\s*{url: "(http[^"]+)".*autoBuffering.*/) {
+ $coded_url = $1;
+ } else {
+ # if we can't get it, just leave as the video URL is there
+ return;
+ }
+
+ debug ("Coded URL: " . $coded_url);
+
+
+ $url = uri_unescape($coded_url);
+ debug("URL: '" . $url . "'");
+
+ # URL ends with filename
+ $name = $url;
+ $name =~ s/.*\/([^\/]+)\?.*/$1/;
+ return $url, title_to_filename($name);
+}
+
+1;
diff --git a/lib/FlashVideo/Site/Vk.pm b/lib/FlashVideo/Site/Vk.pm
new file mode 100644
index 0000000..581784f
--- /dev/null
+++ b/lib/FlashVideo/Site/Vk.pm
@@ -0,0 +1,48 @@
+# Part of get-flash-videos. See get_flash_videos for copyright.
+package FlashVideo::Site::Vk;
+
+use strict;
+use FlashVideo::Utils;
+
+our $VERSION = '0.01';
+sub Version() { $VERSION; }
+
+sub find_video {
+ my ($self, $browser, $embed_url) = @_;
+ my $new_embed_url = "";
+ my $title = "";
+ my $host = "";
+ my $uid = "";
+ my $vtag = "";
+ my $url = "";
+
+ # vkontakte.ru is the same page as vk.com, but it redirects to login (?)
+ if ($embed_url =~ /http:\/\/vkontakte.ru\//) {
+ $embed_url =~ s/http:\/\/vkontakte.ru\//http:\/\/vk.com\//;
+ $browser->get($embed_url);
+ }
+
+ debug ("URI: " . $embed_url);
+
+ if ($browser->content =~ /\s*var video_title = '([^']+)';/) {
+ $title = $1;
+ debug ("Title: '" . $title . "'");
+ }
+
+ return unless ($browser->content =~ /\s*var video_host = '([^']+)';/);
+ $host = $1;
+ debug ("Host: '" . $host . "'");
+
+ return unless ($browser->content =~ /\s*var video_uid = '([^']+)';/);
+ $uid = $1;
+ debug ("UID: '" . $uid . "'");
+
+ return unless ($browser->content =~ /\s*var video_vtag = '([^']+)';/);
+ $vtag = $1;
+
+ $url = $host . "u" . $uid . "/videos/" . $vtag . ".360.mp4";
+ debug ("URL: '" . $url . "'");
+ return $url, title_to_filename($title, "mp4");
+}
+
+1;
diff --git a/lib/FlashVideo/Site/Vkontakte.pm b/lib/FlashVideo/Site/Vkontakte.pm
new file mode 100644
index 0000000..a399a77
--- /dev/null
+++ b/lib/FlashVideo/Site/Vkontakte.pm
@@ -0,0 +1,7 @@
+# Part of get-flash-videos. See get_flash_videos for copyright.
+package FlashVideo::Site::Vkontakte;
+
+use strict;
+use base 'FlashVideo::Site::Vk';
+
+1;
diff --git a/lib/FlashVideo/Site/Wat.pm b/lib/FlashVideo/Site/Wat.pm
index b9cfe5b..f799fef 100644
--- a/lib/FlashVideo/Site/Wat.pm
+++ b/lib/FlashVideo/Site/Wat.pm
@@ -6,18 +6,24 @@
use HTML::Entities;
use URI::Escape;
+our $VERSION = '0.01';
+sub Version { $VERSION; }
+
die "Must have Digest::MD5 for this download\n"
unless eval {
require Digest::MD5;
};
sub token {
- my $url = shift;
- my $hexdate = sprintf("%x",time());
- # fill up triling zeroes
- $hexdate .= "0" x (length($hexdate) - 8);
+ my ($url, $browser) = @_;
+
+ $browser->get("http://www.wat.tv/servertime");
+
+ my $hexdate = $browser->content;
+ my @timestamp = split('\|', $hexdate);
+ $hexdate = sprintf("%x", shift(@timestamp));
my $key = "9b673b13fa4682ed14c3cfa5af5310274b514c4133e9b3a81e6e3aba00912564";
- return Digest::MD5::md5_hex($key . $url . $hexdate)."/".$hexdate;
+ return Digest::MD5::md5_hex($key . $url . $hexdate) . "/" . $hexdate;
}
@@ -33,7 +39,7 @@
my $title = json_unescape(($browser->content =~ /title":"(.*?)",/)[0]);
my $location = "/web/$video_id";
- my $token = &token($location);
+ my $token = &token($location, $browser);
my $url = "http://www.wat.tv/get".$location.
"?token=".$token.
diff --git a/lib/FlashVideo/Site/Yourupload.pm b/lib/FlashVideo/Site/Yourupload.pm
new file mode 100644
index 0000000..39ddd55
--- /dev/null
+++ b/lib/FlashVideo/Site/Yourupload.pm
@@ -0,0 +1,60 @@
+# Part of get-flash-videos. See get_flash_videos for copyright.
+package FlashVideo::Site::Yourupload;
+
+use strict;
+use FlashVideo::Utils;
+use URI::Escape;
+
+our $VERSION = '0.01';
+sub Version() { $VERSION; }
+
+sub find_video {
+ my ($self, $browser, $embed_url) = @_;
+
+ my $flashvars = "";
+ my $file = "";
+ my $url = "";
+ my $name = "";
+
+ if ($embed_url !~ /http:\/\/yourupload.com\/embed\//) {
+ if ($browser->content =~ /<iframe src="(http:\/\/yourupload.com\/embed\/[^"]+)" style/) {
+ $embed_url = $1;
+ $browser->get($embed_url);
+ } else {
+ # we can't find the frame with embed URL
+ return;
+ }
+ }
+
+ # get configuration passed to flash player
+ if ($browser->content =~ /\s*flashvars="([^"]+)"/) {
+ $flashvars = $1;
+ } else {
+ # if we can't get it, just leave as the video URL is there
+ debug("Can't find flashvars");
+ return;
+ }
+
+ debug ("Flashvars: " . $flashvars);
+
+ # in the configuration there is also URL we're looking for
+ if ($flashvars =~ /&file=(http[^&]+)&/) {
+ $file = $1;
+ } else {
+ debug("Can't find file");
+ return;
+ }
+
+ debug("File: " . $file);
+
+ $url = uri_unescape($file);
+ debug("URL: '" . $url . "'");
+
+ # URL ends with filename
+ $name = $url;
+ $name =~ s/.*\/([^\/]+\.(flv|mp4)).*/$1/;
+ debug("Filename: " . $name);
+ return $url, title_to_filename($name);
+}
+
+1;
diff --git a/lib/FlashVideo/Site/Youtube.pm b/lib/FlashVideo/Site/Youtube.pm
index 0abcf15..a3787a5 100644
--- a/lib/FlashVideo/Site/Youtube.pm
+++ b/lib/FlashVideo/Site/Youtube.pm
@@ -8,6 +8,9 @@
use FlashVideo::JSON;
use URI::Escape;
+our $VERSION = '0.01';
+sub Version { $VERSION; }
+
my @formats = (
{ id => 38, resolution => [4096, 2304] },
{ id => 37, resolution => [1920, 1080] },
@@ -68,7 +71,7 @@
my $video_id;
if ($browser->content =~ /(?:var pageVideoId =|(?:CFG_)?VIDEO_ID'?\s*:)\s*'(.+?)'/
|| $browser->content =~ /[&?]video_id=([^&"]+)/
- || $embed_url =~ /v=([^&]+)/
+ || $embed_url =~ /v=([^&#]+)/
|| $browser->content =~ /&video_id=([^&]+)&/) {
$video_id = $1;
} else {
diff --git a/lib/FlashVideo/URLFinder.pm b/lib/FlashVideo/URLFinder.pm
index bd42f4f..caaa894 100644
--- a/lib/FlashVideo/URLFinder.pm
+++ b/lib/FlashVideo/URLFinder.pm
@@ -113,6 +113,9 @@
no strict 'refs';
push @{$package . "::ISA"}, "FlashVideo::Site";
}
+ else {
+ info "Not loading $package $@" if ($@ =~ m%failed%);
+ }
return $package;
}
diff --git a/lib/FlashVideo/Utils.pm b/lib/FlashVideo/Utils.pm
index a519b9b..5eb2973 100644
--- a/lib/FlashVideo/Utils.pm
+++ b/lib/FlashVideo/Utils.pm
@@ -236,7 +236,9 @@
%Win32::API::Type::Known = (int => 'i');
}
- Win32::API->Import("kernel32", "int GetACP()");
+ unless (defined &GetACP) {
+ Win32::API->Import("kernel32", "int GetACP()");
+ }
return "cp" . GetACP();
}
