#!/usr/bin/perl -w # # Copyright G. Westcott - February 2013 # # This code is distributed under the GNU General Public License v2 (GPLv2) . # # For extended help information run # tv_grab_uk_tvguide --info # eval 'exec /usr/bin/perl -w -S $0 ${1+"$@"}' if 0; # not running under some shell use Data::Dumper; use strict; use warnings; use XMLTV 1.0.1; use XMLTV::ProgressBar; use XMLTV::Options qw/ParseOptions/; use XMLTV::Configure::Writer; use XMLTV::Supplement 0.005065 qw/SetSupplementRoot GetSupplementDir GetSupplementLines/; use XMLTV::Get_nice 0.005065 qw/get_nice_tree/; use File::Path; use POSIX qw(strftime); use DateTime; use Date::Parse; #v1.3: use DateTime::Format::DateParse; use Encode; use URI::Escape; use LWP::UserAgent; use HTTP::Cookies; use HTML::TreeBuilder; use HTTP::Cache::Transparent; #require HTTP::Cookies; #my $cookies = HTTP::Cookies->new; #$XMLTV::Get_nice::ua->cookie_jar($cookies); # Although we use HTTP::Cache::Transparent, this undocumented --cache # option for debugging is still useful since it will _always_ use a # cached copy of a page, without contacting the server at all. # use XMLTV::Memoize; XMLTV::Memoize::check_argv('XMLTV::Get_nice::get_nice_aux'); use subs qw(debug warning); my $warnings = 0; use XMLTV::Usage < $GRABBER_NAME, capabilities => [qw/baseline manualconfig apiconfig cache tkconfig/], # apiconfig is needed for --list-channels (bug) stage_sub => \&config_stage, listchannels_sub => \&fetch_channels, version => $VERSION, description => $GRABBER_DESC, extra_options => [qw/nodetailspage legacychannels method:i makeignorelist:s useignorelist:s usage test:i/], }); usage(1) if $opt->{usage}; #print Dumper($conf); exit; # any overrides? if (defined( $conf->{'generator-info-name'} )) { $generator_info_name = $conf->{'generator-info-name'}->[0]; } if (defined( $conf->{'generator-info-url'} )) { $generator_info_url = $conf->{'generator-info-url'}->[0]; } if (defined( $conf->{'source-info-name'} )) { $source_info_name = $conf->{'source-info-name'}->[0]; } if (defined( $conf->{'source-info-url'} )) { $source_info_url = $conf->{'source-info-url'}->[0]; } # initialise the gui XMLTV::Ask::init($opt->{gui}); # ------------------------------------------------------------------------------------------------------------------------------------- # # Initialise the web page cache init_cachedir( $conf->{cachedir}->[0] ); HTTP::Cache::Transparent::init( { BasePath => $conf->{cachedir}->[0], NoUpdate => 60*60, # cache time in seconds MaxAge => 24, # flush time in hours Verbose => $opt->{debug}, } ); # ------------------------------------------------------------------------------------------------------------------------------------- # # Check we have all our required conf params config_check(); # Load the conf file containing mapped channels and categories information my %mapchannelhash; my %mapcategoryhash; loadmapconf(); #print Dumper(\%mapchannelhash, \%mapcategoryhash); exit; # ------------------------------------------------------------------------------------------------------------------------------------- # # Progress Bar :) my $bar = new XMLTV::ProgressBar({ name => "Fetching listings", count => (scalar @{$conf->{channel}}) * ($opt->{days} + 1) # +1 added for the extra day necessary for <06:00 programmes }) unless ($opt->{quiet} || $opt->{debug}); # ------------------------------------------------------------------------------------------------------------------------------------- # # Data store before being written as XML my $programmes = (); my $channels = (); # Store channel names during fetch my $channames = undef; # Cache for alternative ID lookups my $channels_cache = {}; my $channels_alt = {}; my $channels_alt_found = {}; # Get the schedule(s) from TV Guide fetch_listings(); # print Dumper($programmes); # Progress Bar $bar->finish() && undef $bar if defined $bar; # ------------------------------------------------------------------------------------------------------------------------------------- # # Filter out programmes outside of requested period (see man page) my %w_args; if (($opt->{offset} != 0) || ($opt->{days} != -999)) { $w_args{offset} = $opt->{offset}; $w_args{days} = ($opt->{days} == -999) ? 100 : $opt->{days}; $w_args{cutoff} = '000000'; # e.g. '060000' } # ------------------------------------------------------------------------------------------------------------------------------------- # # Generate the XML my $encoding = 'UTF-8'; my $credits = { 'generator-info-name' => $generator_info_name, 'generator-info-url' => $generator_info_url, 'source-info-name' => $source_info_name, 'source-info-url' => $source_info_url }; XMLTV::write_data([ $encoding, $credits, $channels, $programmes ], %w_args); # Finished! # ------------------------------------------------------------------------------------------------------------------------------------- # # Signal that something went wrong if there were warnings. exit(1) if $warnings; # All data fetched ok. debug "Exiting without warnings."; exit(0); # ############################################################################# # # THE MEAT ##################################################################### # ------------------------------------------------------------------------------------------------------------------------------------- # sub fetch_listings { # Fetch listings per channel foreach my $channel_id (@{$conf->{channel}}) { # Officially: # http://www.tvguide.co.uk/channellisting.asp?cTime=3%2F19%2F2013+06%3A00%3A00+&ch=857&go=go # But this works too: # http://www.tvguide.co.uk/channellisting.asp?ch=86&cTime=3/18/2013 my $baseurl = $ROOT_URL.'channellistings.asp'; debug('+' x 80); # Now grab listings for each channel on each day, according to the options in $opt # # tvguide runs from 06:00 so we need to get the previous day as well just for any programmes after midnight # for (my $i=($opt->{offset} -1); $i < ($opt->{offset} + $opt->{days}); $i++) { # Inner loop allows us to push alternative IDs (if found) into @alts when no schedule is found for the user selected ID my @alts = int($channel_id); # Have we already found a working alternative ID for the user's selected ID if (defined $channels_alt_found->{$channel_id}) { @alts = $channels_alt_found->{$channel_id}; debug "Using alternative ID: ".$channels_alt_found->{$channel_id}.' for '.$channel_id; } foreach my $alt_channel_id (@alts) { my $alt_success = 0; my $theday = DateTime->today->add (days => $i)->set_time_zone('Europe/London'); # Construct the listings url my $url = $baseurl . '?ch=' . $alt_channel_id . '&cTime=' . uri_escape( $theday->strftime('%m/%d/%Y 00:00:00') ); #debug "Fetching: $url"; # If we need to map the fetched channel_id to a different value my $xmlchannel_id = $channel_id; $xmlchannel_id .= '.tvguide.co.uk' unless $opt->{legacychannels}; # make channel RFC2838 compliant if (defined(&map_channel_id)) { $xmlchannel_id = map_channel_id($xmlchannel_id); } # Fetch the page # my $tree = XMLTV::Get_nice::get_nice_tree($url); my $tree = fetch_url($url); # $tree->dump; exit; # Scrub the page if ($tree) { my $channelname = undef; # Store the channel ids in a list (do this only once per program run) if (!defined $channames) { #debug 'fetching options tags'; my $choptions = $tree->look_down('_tag' => 'select', 'name' => 'ch'); if (defined $choptions) { my @choptionslist = $choptions->look_down('_tag' => 'option'); if (@choptionslist) { foreach my $choption (@choptionslist) { $channames->{$choption->attr('value')} = $choption->as_text; } } } } $channelname = $channames->{$alt_channel_id} if $channames; # Try a fallback method if the form options are missing [Credit mkbloke] if (!defined $channelname) { #debug 'using fallback method'; my $fallback = $tree->look_down('_tag' => 'input', 'name' => 'cTime'); $fallback = $fallback->look_up('_tag', 'tr') if $fallback; $channelname = $fallback->look_down('_tag' => 'span', 'class' => 'programmeheading') if $fallback; $channelname = $channelname->as_text if $channelname; } #debug 'found channel name: '.$channelname; # tvguide website can be very slow - try to avoid barfing when no response # if no channelname then assume we got no response from website if (!defined $channelname) { warning "Unable to retrieve web page for $alt_channel_id"; next; } # # my @shows = $tree->look_down('_tag' => 'table', 'border' => '0', 'cellpadding' => '0', 'style' => qr/background:\s*black;border-collapse:\s*collapse;/); if (@shows) { my $count = 0; foreach my $show (@shows) { # $show->dump; $count++; # are we processing yesterday's schedule? (see above) if ($i == ($opt->{offset} -1)) { my $showstart = $show->look_down('_tag' => 'span', 'class' => 'tvchannel'); my ($h, $i, $a) = $showstart->as_text =~ /(\d*):(\d*)\s*(am|pm)/; # 2014-04-02 see note below if (!defined $a) { $showstart = $show->look_down('_tag' => 'span', 'class' => 'season'); ($h, $i, $a) = $showstart->as_text =~ /(\d*):(\d*)\s*(am|pm)/; } if ($a eq 'am' && ($h < 6 || $h == 12)) { next if $count == 1; # we don't want first programme in file if it overlaps 6am boundary # continue processing of pre-6am programme } else { next; } $showstart = $h = $i = $a = undef; } my %prog = (); my $showtime; # see if we have a details page # look_down('_tag' => 'a', 'href' => qr/javascript:popup/); # 2014-12-03 The new website seems a bit flakey with these details pages, often returning a 500 Server Error # Here's an option to disable the details pages ( --nodetailspage ) # 2014-12-24 site changed # my $webdetails = $show->look_down('_tag' => 'a', 'href' => qr/\/engage\//); # if (!$opt->{nodetailspage}) { my $webdetails = $show->look_down('_tag' => 'a', 'href' => qr/\/detail\//); my $href = $webdetails->attr('href'); # my ($id) = $href =~ /javascript:popup\((\d*)\);/; # $url = $ROOT_URL . 'detail.asp?id=' . $id; # my ($url) = $href =~ /javascript:popupshow\('(.*)'\);/; my ($url) = $href; #debug "Fetching: $url"; # Fetch the page # my $showdetail = XMLTV::Get_nice::get_nice_tree($url); my $showdetail = fetch_url($url); # $showdetail->dump; if ($showdetail) { # Details page contains Director names and a better list of Actors # Get the cast and extract them into a new tree my @lis = $showdetail->look_down('_tag' => 'div', 'class' => 'cast-entry'); LOOP: foreach my $person (@lis) { # # 30/6/16 #
# Margaret Sellinger # # # Lesley-Anne Down # # # (IMDB) # #
my ($name, $role); if ( my ($_name) = $person->look_down('_tag' => 'span', 'itemprop' => 'name') ) { $name = $_name->as_text; } if ( my ($_role) = $person->look_down('_tag' => 'span', 'class' => 'role') ) { $role = $_role->as_text; } # drop the "Executive Director" & "Executive Producer" - any others we should drop? next LOOP if ( $role =~ /^(Executive Director|Executive Producer)/ ); # map the website role to an xmltv role my %xmltvroles = ( 'Director'=>'director', 'Producer'=>'producer', 'Series Producer'=>'producer', 'Writer'=>'writer', 'Co-Director'=>'director', 'Presenter'=>'presenter', 'Commentator'=>'commentator', 'Guest'=>'guest' ); my $credit; if (exists $xmltvroles{$role}) { $credit = $xmltvroles{$role}; } else { $credit = 'actor'; } if ($credit eq 'actor' && defined $role) { push @{$prog{'credits'}{$credit}}, [ encode('utf-8', $name), encode('utf-8', $role) ]; } else { push @{$prog{'credits'}{$credit}}, encode('utf-8', $name); } } undef @lis; # Get the "Left Panel" which contains the programme times and attributes my $lhs = $showdetail->look_down('_tag' => 'div', 'class' => qr/divLHS-section-2/); # Get the programme's "attributes" e.g. "Certificate" if ($lhs) { my @attrs = $lhs->look_down('_tag' => 'span', 'class' => 'LHS-attribute'); if (@attrs) { foreach my $attr (@attrs) { # $attr->dump; if ( my $showattr = $attr->as_text() ) { if ( $showattr =~ /^Certificate\s:\s(.*)\s*$/ ) { $prog{'rating'} = [[ $1, 'BBFC' ]] if $1; } } } } } # start time, and stop time (actually an optional DTD element) # 10:00am-11:50am (1 hour 50 minutes) Wed 20 Mar # (use the Date provided to avoid issues with the site running from 06:00-06:00) # # Note site displays stop time wrong on GMT/BST changeover, e.g.: # 12:45am-1:10am (25 minutes) Sun 31 Mar # this should be 12:45am-2:10am (BST) # this makes $showtime->set barf on "invalid local time for date in timezone" # # 1/Jan/17 times are now in the left panel. 'datetime' is used for user comments! # my $showtimes = $showdetail->look_down('_tag' => 'span', 'class' => 'datetime'); # # 25/Mar/2023 now displays the correct time on GMT/BST changeover...but the duration is wrong! # Sun 26 Mar 12:40am-2:40am (2 hours) # this should be '1 hour' # if ($lhs) { # Unfortunately the div with the date doesn't have any safe identifier. There are several ways we could remove the # cruft from the container but the following, although clunky, is probably the safest my ($dt, $h, $i, $a, $h2, $i2, $a2) = $lhs->as_text =~ /((?:Mon|Tue|Wed|Thu|Fri|Sat|Sun|Christmas\s(?:Eve|Day)|Boxing\sDay|New\sYears\s(?:Eve|Day))[\s<].*?)(\d*):(\d*)(am|pm)(?:-(\d*):(\d*)(am|pm))?/; # print STDERR $dt."\n"; if ($dt && $dt !~ /\D\D\D\s\d\d?\s\D\D\D/) { my @thedt = localtime(time); # ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) my ($yr1, $yr2) = ($thedt[5]+1900, $thedt[5]+1900); if ($thedt[4] == 11) { $yr2++; } if ($thedt[4] == 0) { $yr1--; } SWITCH: { $dt =~ 'Christmas\s+Eve' && do { $dt = '24 Dec '.$yr1; last SWITCH; }; $dt =~ 'Christmas\s+Day' && do { $dt = '25 Dec '.$yr1; last SWITCH; }; $dt =~ 'Boxing\s+Day' && do { $dt = '26 Dec '.$yr1; last SWITCH; }; $dt =~ 'New\s+Years\s+Eve' && do { $dt = '31 Dec '.$yr1; last SWITCH; }; $dt =~ 'New\s+Years\s+Day' && do { $dt = '1 Jan ' .$yr2; last SWITCH; }; undef $dt; } } if ($dt) { #v1.3: $showtime = DateTime::Format::DateParse->parse_datetime( $dt, 'Europe/London' ); # workaround for bug in Date::Time::str2time() which generates wrong dates for future months when no year is given # https://rt.cpan.org/Public/Bug/Display.html?id=92611 $dt .= ' '.( $theday->year() ) if $dt !~ /(19|20)\d\d/; $showtime = DateTime->from_epoch( epoch=>str2time( $dt, 'GMT' ) )->set_time_zone('Europe/London'); } else { $showtime = $theday->clone; } $h += 12 if $a eq 'pm' && $h < 12; # e.g. 12:30pm means 12:30 ! $h -= 12 if $a eq 'am' && $h == 12; # e.g. 12:30am means 00:30 ! $h2 += 12 if $a2 eq 'pm' && $h2 < 12; $h2 -= 12 if $a2 eq 'am' && $h2 == 12; $showtime->set(hour => $h, minute => $i, second => 0); $prog{'start'} = $showtime->strftime("%Y%m%d%H%M%S %z"); my $showtime_ = $showtime->clone; if (defined $h2 && $h2 >= 0) { $showtime->add (days => 1) if $h2 < $h; # see note above re errors with GMT/BST transition eval { # try $showtime->set(hour => $h2, minute => $i2, second => 0); $prog{'stop'} = $showtime->strftime("%Y%m%d%H%M%S %z"); } or do { # catch # let's see if we can get a duration my ($durh, $durm) = $lhs->as_text =~ /\((?:(\d*)\shours?)?\s?(?:(\d*)\sminutes?)?\)/; if (defined $durh || defined $durm) { $durh = 0 if !defined $durh; $durm = 0 if !defined $durm; $showtime_->set_time_zone('UTC')->add( hours => $durh, minutes => $durm )->set_time_zone('Europe/London'); $prog{'stop'} = $showtime_->strftime("%Y%m%d%H%M%S %z"); } else { # no output prog 'stop' time } } } else { # no output prog 'stop' time } # if the first programme starts before 06:00 we should ignore it as it will create a duplicate programme if ($h < 6) { # skip this prog as we have already retrieved it with 'yesterday's schedule next if ($count == 1); } } # programme image my $c = $showdetail->look_down('_tag' => 'div', 'id' => 'divCentre'); if ($c) { my $img = $c->look_down('_tag' => 'div', 'id' => 'headerImage'); if ($img) { my $showattr = $img->attr('style'); (my $showimage) = $showattr =~ /background-image:\s*url\((.*?\.jpg)\)/; if ($showimage) { $prog{'image'} = [[ $showimage, { 'system'=>'tvguide', 'type'=>'backdrop' } ]]; } } } } # end showdetail $showdetail->delete() if $showdetail; } # end nodetailspage # channel $prog{'channel'} = $xmlchannel_id; # title (mandatory) # Baywatch my $showtitle = $show->look_down('_tag' => 'span', 'class' => 'programmeheading'); $prog{'title'} = [[ encode('utf-8', $showtitle->as_text), 'en' ]]; $showtitle->detach; # Note: is used by StartTime then SubTitle then Category then Subtitles/B&W/etc # start (mandatory) # 3:00pm # (don't add it even we already have it from the detail page but we still need to delete it from the tree) # @ 2014-04-02 the site has changed to # 6:00 am # but this just doesn't sound right to me (i.e. I think it might change again), so let's try both ways # # @2016-08-05 looks like this is permanent # my $showstart = $show->look_down('_tag' => 'span', 'class' => 'tvchannel'); my $showstart = $show->look_down('_tag' => 'span', 'class' => 'season'); if (!$prog{'start'}) { my ($h, $i, $a) = $showstart->as_text =~ /(\d*):(\d*)\s*(am|pm)/; if (!defined $a) { $showstart = $show->look_down('_tag' => 'span', 'class' => 'season'); ($h, $i, $a) = $showstart->as_text =~ /(\d*):(\d*)\s*(am|pm)/; } $h += 12 if $a eq 'pm' && $h < 12; # e.g. 12:30pm means 12:30 ! $h -= 12 if $a eq 'am' && $h == 12; # e.g. 12:30am means 00:30 ! $showtime = $theday->clone; # @ 2023-03-25 The assumption that the site works from 06:00 - 06:00 breaks when the first programme overlaps 06:00 # so it seems the rule is the 'first' programme ends *after* 6:00 am # Saturday, March 25, 2023 ITVBe # 1:00am Teleshopping # 7:00am Sam Faiers: The Mummy Diaries # # Saturday, March 25, 2023 BBC Two HD # 3:45am This Is BBC Two # 6:35am Hey Duggee # Only way I can think of to handle this is to tag the first programme on a day # In addition, this programme should be ignored to avoid duplicates # if ($h < 6) { # skip this prog as we have already retrieved it with 'yesterday's schedule next if ($count == 1); $showtime->add(days => 1) if ($h < 6); # site runs from 06:00-06:00 so anything <06:00 is for tomorrow } $showtime->set(hour => $h, minute => $i, second => 0); $prog{'start'} = $showtime->strftime("%Y%m%d%H%M%S %z"); # no prog 'stop' time available } $showstart->detach; # category # Category General Movie/Drama my $showcategory = $show->look_down('_tag' => 'span', 'class' => 'tvchannel', sub { $_[0]->as_text =~ /^Category\s*$/ } ); if ($showcategory) { $showcategory = $showcategory->right; my @showcategory = split(/\//, $showcategory->as_text); my @showcategories = (); foreach my $category (@showcategory) { # category translation? if (defined(&map_category)) { $category = map_category($category); } if ($category =~ /\|/) { foreach my $cat (split(/\|/, $category)) { push @showcategories, $cat unless grep(/$cat/, @showcategories); } } elsif ($category ne '') { push @showcategories, $category unless grep(/$category/, @showcategories); } } foreach my $category (@showcategories) { push @{$prog{'category'}}, [ encode('utf-8', $category), 'en' ]; } $showcategory->left->detach; } # desc # Dissolving bikinis cause a stir on the beach my $showdesc = $show->look_down('_tag' => 'span', 'class' => 'programmetext'); if ($showdesc) { $showdesc = $showdesc->as_text; $showdesc .= '.' if ( (length $showdesc) && ((substr $showdesc,-1,1) ne '.') ); # append a fullstop if (length $showdesc) { $prog{'desc'} = [[ encode('utf-8', $showdesc), 'en' ]]; } } # year # strip this off the title e.g. "A Useful Life (2010)" my ($showyear) = $prog{'title'}->[0][0] =~ /.*\((\d\d\d\d)\)$/; if ($showyear) { $prog{'date'} = $showyear; # assume anything with a year is a film - add Films category group push @{$prog{'category'}}, [ 'Films', 'en' ]; } # flags # (Subtitles) (Black & White) my $showflags = $show->look_down('_tag' => 'span', 'class' => 'tvchannel', sub { $_[0]->as_text =~ /Subtitles/ } ); if ($showflags) { push @{$prog{'subtitles'}}, {'type' => 'teletext'}; $showflags->detach; } $showflags = $show->look_down('_tag' => 'span', 'class' => 'tvchannel', sub { $_[0]->as_text =~ /Audio Described/ } ); if ($showflags) { # push @{$prog{'subtitles'}}, {'type' => 'deaf-signed'}; <-- Audio Described is not deaf-signed $showflags->detach; } $showflags = $show->look_down('_tag' => 'span', 'class' => 'tvchannel', sub { $_[0]->as_text =~ /Repeat/ } ); if ($showflags) { # push @{$prog{'previously-shown'}}, {}; $prog{'previously-shown'} = {}; $showflags->detach; } my $showvideo = $show->look_down('_tag' => 'span', 'class' => 'tvchannel', sub { $_[0]->as_text =~ /Black & White/ } ); if ($showvideo) { $prog{'video'}->{'colour'} = '0'; $showvideo->detach; } #if ($showflags && $showflags->as_text =~ '\[REP\]') { # push @{$prog{'previously-shown'}}, {}; #} $showflags = $show->look_down('_tag' => 'span', 'class' => 'tvchannel', sub { $_[0]->as_text =~ /Interactive/ } ); if ($showflags) { # no flag in DTD for this $showflags->detach; } # episode number # Season 2 Episode 3 of 22 my @showepisode = $show->look_down('_tag' => 'span', 'class' => 'season'); my $showepisode; foreach my $el (@showepisode) { $showepisode .= $el->as_text; } if ($showepisode) { my ($showsxx, $showexx, $showeof) = ( $showepisode =~ /^(?:(?:Series|Season) (\d+)(?:[., :]+)?)?(?:Episode (\d+)(?: of (\d+))?)?/ ); # scan the description for any "Part x of x." info my ($showpxx, $showpof) = ('', ''); ($showpxx, $showpof) = ( $showdesc =~ /Part (one|two|three|four|five|six|seven|eight|nine|\d+)(?: of (one|two|three|four|five|six|seven|eight|nine|\d+))?/ ) if ($showdesc); my $showepnum = make_ns_epnum($showsxx, $showexx, $showeof, $showpxx, $showpof); if ($showepnum && $showepnum ne '...') { $prog{'episode-num'} = [[ $showepnum, 'xmltv_ns' ]]; } #debug "--$showepnum-- ".$showepisode->as_text; } # episode title # The Fabulous Buchannon Boys my $showeptitle = $show->look_down('_tag' => 'span', 'class' => 'tvchannel'); if ($showeptitle) { if ($showeptitle->as_text =~ /\(?Premiere\)?/) { $prog{'premiere'} = []; } else { $prog{'sub-title'} = [[ encode('utf-8', $showeptitle->as_text), 'en' ]]; } $showeptitle->detach; } # director # never seen one but let's assume they're in the description if (!$prog{'credits'}->{'director'}) { if ($showdesc) { my ($directors) = ( $showdesc =~ /(?:Directed by|Director) ([^\.]*)\.?/ ); if ($directors) { $directors =~ s/ (with|and) /,/ig; $directors =~ s/ (singer|actor|actress) //ig; # strip these words $directors =~ s/,,/,/g; # delete empties $directors = encode('utf-8', $directors); # encode names into utf-8 #debug $directors; my @directors = split(/,/, $directors); s{^\s+|\s+$}{}g foreach @directors; # strip leading & trailing spaces $prog{'credits'}->{'director'} = \@directors if (scalar @directors > 0); } } } # actors # these are buried in the description :-( if (!$prog{'credits'}->{'actor'}) { if ($showdesc) { my ($actors) = ( $showdesc =~ /(?:starring)([^\.]*)\.?/i ); if ($actors) { $actors =~ s/ (also|starring|with|and) /,/ig; # may be used to separate names $actors =~ s/ (singer|actor|actress) //ig; # strip these words $actors =~ s/,,/,/g; # delete empties $actors = encode('utf-8', $actors); # encode names into utf-8 #debug $actors; my @actors = split(/,/, $actors); s{^\s+|\s+$}{}g foreach @actors; # strip leading & trailing spaces $prog{'credits'}->{'actor'} = \@actors if (scalar @actors > 0); } } } # rating # Rating
3.9 my $showrating = $show->look_down('_tag' => 'span', 'class' => 'programmetext', sub { $_[0]->as_trimmed_text =~ /^Rating$/ } ); if ($showrating) { $showrating = $showrating->right; $showrating = $showrating->right if ($showrating->tag eq 'br'); if ($showrating->tag eq 'span' && $showrating->attr('class') eq 'programmeheading') { if ($showrating->as_text) { $prog{'star-rating'} = [ $showrating->as_text . '/10' ]; } } } # programme url # my $showurl = $show->look_down('_tag' => 'a', 'title' => 'Click to rate and review'); if ($showurl) { $prog{'url'} = [ encode( 'utf-8', $showurl->attr('href') ) ]; } # programme image #
my $showattr = $show->attr('style'); (my $showimage) = $showattr =~ /background-image:\s*url\((.*?\.jpg)\)/; if ($showimage) { $prog{'image'} = [] if not defined $prog{'image'} or not @{$prog{'image'}}; push @{$prog{'image'}}, [ $showimage, { 'system'=>'tvguide', 'type'=>'backdrop' } ]; } # debug Dumper \%prog; push(@{$programmes}, \%prog); $alt_success = 1; } # if this is an alternative id then remember it if ($alt_success) { if ($alt_channel_id != $channel_id) { $channels_alt_found->{$channel_id} = $alt_channel_id; debug('Found working alternative ID '.$alt_channel_id.' for '.$channel_id); } } } else { # no schedule found debug "No schedule found for channel ID: $alt_channel_id"; # append alternative channel numbers to list if (scalar @alts == 1) { push @alts, get_alt_channel_ids($channel_id, $channelname); debug "Found alternative IDs: @alts[1..$#alts]" if (scalar @alts > 1); } # issue warning only when all alternatives have been tried warning 'No schedule found' if ($alt_channel_id == $alts[$#alts]); } undef @shows; # Add to the channels hash $channels->{$channel_id} = { 'id'=> $xmlchannel_id , 'display-name' => [[ encode('utf-8', $channelname), 'en' ]] }; $tree->delete(); } else { # tree conversion failed warning 'Could not parse the page'; } last if $alt_success; } $bar->update if defined $bar; debug('-' x 30); } } } # ############################################################################# # # THE VEG ###################################################################### # ------------------------------------------------------------------------------------------------------------------------------------- # sub make_ns_epnum { # Convert an episode number to its xmltv_ns compatible - i.e. reset the base to zero # Input = series number, episode number, total episodes, part number, total parts, # e.g. "1, 3, 6, 2, 4" >> "0.2/6.1/4", "3, 4" >> "2.3." # my ($s, $e, $e_of, $p, $p_of) = @_; # debug Dumper(@_); # "Part x of x" may contain integers or words (e.g. "Part 1 of 2", or "Part one") $p = text_to_num($p) if defined $p; $p_of = text_to_num($p_of) if defined $p_of; # re-base the series/episode/part numbers $s-- if (defined $s && $s > 0); $e-- if (defined $e && $e > 0); $p-- if (defined $p && $p && $p=~/^\d+$/ && $p > 0); # make the xmltv_ns compliant episode-num my $episode_ns = ''; $episode_ns .= $s if defined $s; $episode_ns .= '.'; $episode_ns .= $e if defined $e; $episode_ns .= '/'.$e_of if defined $e_of; $episode_ns .= '.'; $episode_ns .= $p if $p; $episode_ns .= '/'.$p_of if $p_of; #debug "--$episode_ns--"; return $episode_ns; } sub text_to_num { # Convert a word number to int e.g. 'one' >> '1' # my ($text) = @_; if ($text !~ /^[+-]?\d+$/) { # standard test for an int my %nums = (one => 1, two => 2, three => 3, four => 4, five => 5, six => 6, seven => 7, eight => 8, nine => 9); return $nums{$text} if exists $nums{$text}; } return $text } sub map_channel_id { # Map the fetched channel_id to a different value (e.g. our PVR needs specific channel ids) # mapped channels should be stored in a file called tv_grab_uk_tvguide.map.conf # containing lines of the form: map==fromchan==tochan e.g. 'map==109==BBC4' # my ($channel_id) = @_; my $mapchannels = \%mapchannelhash; if (%mapchannelhash && exists $mapchannels->{$channel_id}) { return $mapchannels->{$channel_id} ; } return $channel_id; } sub map_category { # Map the fetched category to a different value (e.g. our PVR needs specific genres) # mapped categories should be stored in a file called tv_grab_uk_guardian.map.conf # containing lines of the form: cat==fromcategory==tocategory e.g. 'cat==General Movie==Film' # my ($category) = @_; my $mapcategories = \%mapcategoryhash; if (%mapcategoryhash && exists $mapcategories->{$category}) { return $mapcategories->{$category} ; } return $category; } sub loadmapconf { # Load the conf file containing mapped channels and categories information # # This file contains 2 record types: # lines starting with "map" are used to 'translate' the incoming channel id to those required by your PVR # e.g. map==dave==DAVE will output "DAVE" in your XML file instead of "dave" # lines starting with "cat" are used to translate categories (genres) in the incoming data to those required by your PVR # e.g. cat==Science Fiction==Sci-fi will output "Sci-Fi" in your XML file instead of "Science Fiction" # my $mapchannels = \%mapchannelhash; my $mapcategories = \%mapcategoryhash; # my $supplementdir = $ENV{XMLTV_SUPPLEMENT} || GetSupplementDir(); # get default file from supplement.xmltv.org if local file not exist if (-f File::Spec->catfile( $supplementdir, $GRABBER_NAME, $GRABBER_NAME . '.map.conf' ) ) { SetSupplementRoot($supplementdir); } my $lines = GetSupplementLines($GRABBER_NAME, $GRABBER_NAME . '.map.conf'); foreach my $line (@$lines) { my ($type, $mapfrom, $mapto, $trash) = $line =~ /^(.*)==(.*)==(.*?)([\s\t]*#.*)?$/; SWITCH: { lc($type) eq 'map' && do { $mapchannels->{$mapfrom} = $mapto; last SWITCH; }; lc($type) eq 'cat' && do { $mapcategories->{$mapfrom} = $mapto; last SWITCH; }; warning "Unknown type in map file: \n $line"; } } # debug Dumper ($mapchannels, $mapcategories); } sub fetch_all_channel_ids { # Fetch all channel IDs with method 1, used for channel list creation and alternative ID searches # my $channels = {}; my $tree = fetch_url('https://www.tvguide.co.uk/mychannels.asp?gw=1242', 'post', [ thisDay => '', thisTime => '', gridSpan => '', emailaddress => '', regionid => 1, systemid => 5, xn => 'Show me the channels' ]); my @c = $tree->look_down('_tag' => qr/table|tr/, 'class' => qr/^tr[XC]/); my $j = 0 if $opt->{test}; # --test is an undocumented (private) option foreach (@c) { my ($ch, $id, $l, $t); if ($_->id =~ /^trX?\d+/) { ($id) = $_->id =~ /^trX?(\d+)/; ($ch) = $ROOT_URL.'channellistings.asp?ch='.$id; ($l) = $_->as_HTML =~ /background-image:url\(([^)]+)\)/; ($t) = $_->as_text; } $channels->{$id} = {id => $id . (!$opt->{'list-channels'}?" # ".encode('utf-8', $t):(!$opt->{legacychannels}?'.tvguide.co.uk':'')), 'display-name' => [[ encode('utf-8', $t), 'en' ]], icon => [{ 'src'=>$l }], url => [ $ch ], } if $id; debug $id if $opt->{test}; last if $opt->{test} and (++$j >= $opt->{test}); # limit during testing } return $channels; } sub get_alt_channel_ids ($$) { # Find alternate IDs for the same exact channel name # my ($channel_id, $channel_name) = @_; if (defined $channels_alt->{$channel_id}) { return @{$channels_alt->{$channel_id}}; } $channels_cache = fetch_all_channel_ids() if (scalar keys %$channels_cache == 0); my @alts; foreach my $id (keys %$channels_cache) { next if $id == $channel_id; push @alts, $id if ($channels_cache->{$id}->{'display-name'}[0][0] eq encode('utf-8', $channel_name) ); } $channels_alt->{$channel_id} = \@alts; return @{$channels_alt->{$channel_id}}; } sub fetch_channels { # ParseOptions() handles --configure and --list-channels internally without returning, # so we do not have global $opt available during --configure # unless we copy vars to main package. (Note package variable must be declared with 'our') # ($main::conf, $main::opt) = @_; my $channels = {}; # Get the list of available channels #------------------------------------------------------------------------------------------- # preferred method (uses a html select list of channels) # if ((scalar keys %$channels == 0) && (!defined $opt->{method} || $opt->{method} == 0)) { my $bar = new XMLTV::ProgressBar({ name => "Fetching channels", count => 1 }) unless ($opt->{quiet} || $opt->{debug}); # Fetch channels via a dummy call to BBC1 listings # my $channel_list = $ROOT_URL.'channellistings.asp?ch=74&cTime='; my @channels; my $tree = XMLTV::Get_nice::get_nice_tree($channel_list); # $tree->dump; my $_channels = $tree->look_down('_tag' => 'select', 'name' => 'ch'); if (defined $_channels) { @channels = $_channels->look_down('_tag' => 'option'); # debug $_channels->as_HTML; # foreach my $xchannel (@channels) { debug $xchannel->as_HTML; } } $bar->update() if defined $bar; $bar->finish() && undef $bar if defined $bar; if (scalar @channels > 0) { $bar = new XMLTV::ProgressBar({ name => "Parsing result", count => scalar @channels }) unless ($opt->{quiet} || $opt->{debug}); # Browse through the downloaded list of channels and map them to a hash XMLTV::Writer would understand foreach my $channel (@channels) { if ($channel->as_text) { my ($id) = $channel->attr('value'); my ($url) = 'channellistings.asp?ch=' . $channel->attr('value'); my ($name) = $channel->as_text; $channels->{$id} = { id => $id . (!$opt->{'list-channels'}?" # ".encode('utf-8', $name):(!$opt->{legacychannels}?'.tvguide.co.uk':'')), 'display-name' => [[ encode('utf-8', $name), 'en' ]], url => [ $ROOT_URL.$url ] }; } $bar->update() if defined $bar; } $bar->finish() && undef $bar if defined $bar; } warning "No channels found in TVGuide" if (scalar keys %$channels == 0); } #------------------------------------------------------------------------------------------- # workaround for broken TVGuide website [2022-01-26] # # alternative Method 1 (credit: mkbloke ) # # if ((scalar keys %$channels == 0) && (!defined $opt->{method} || $opt->{method} == 1)) { # alternative method 1 # fetches channels from the website's "mychannels" page # (preferred method found 899, this method finds 939 ) # (as at 2023-01-25 it finds 1207 channels, but many are duplicates with no data) warning "Trying alternative method 1"; my $bar = new XMLTV::ProgressBar({ name => "Fetching channels", count => 1 }) unless ($opt->{quiet} || $opt->{debug}); $channels = fetch_all_channel_ids(); $bar->update() if defined $bar; $bar->finish() && undef $bar if defined $bar; warning "Found ".(scalar keys %$channels)." channels"; } #------------------------------------------------------------------------------------------- # alternative Method 2 # # if ((scalar keys %$channels == 0) && (!defined $opt->{method} || $opt->{method} == 2)) { # alternative method 2 # fetches channels from the website's mobile-friendly page # (preferred method found 899, this method finds 387 ) warning "Trying alternative method 2"; my $bar = new XMLTV::ProgressBar({ name => "Fetching channels", count => 12 }) unless ($opt->{quiet} || $opt->{debug}); foreach (qw/ 7 3 12 5 25 8 22 19 10 29 18 23 /) { my $url = 'https://www.tvguide.co.uk/mobile/?systemid='.$_; #debug "Fetching: $url"; my $tree = XMLTV::Get_nice::get_nice_tree($url); my @c = $tree->look_down('_tag' => 'div', 'class' => 'div-channel-progs'); foreach (@c) { my ($ch, $id, $l, $t); $ch = $_->look_down('_tag' => 'a')->attr('href'); ($id) = $ch =~ m/\?ch=(\d+)$/; my $_dl = $_->look_down('_tag' => 'div', 'class' => 'div-channel-logo'); if ($_dl) { # don't chain the look_downs (for better robustness) my $_il = $_dl->look_down('_tag' => 'img', 'class' => 'img-channel-logo'); if ($_il) { $l = $_il->attr('src'); $t = $_il->attr('alt'); $t =~ s/ TV Listings$//; } } $ch =~ s/mobile\/channellisting\.asp/channellistings\.asp/; $channels->{$id} = {id => $id . (!$opt->{'list-channels'}?" # ".encode('utf-8', $t):(!$opt->{legacychannels}?'.tvguide.co.uk':'')), 'display-name' => [[ encode('utf-8', $t), 'en' ]], icon => [{ 'src'=>$l }], url => [ $ch ], } if $id; } $bar->update() if defined $bar; } $bar->finish() && undef $bar if defined $bar; warning "Found ".(scalar keys %$channels)." channels \n"; } #------------------------------------------------------------------------------------------- # does user want to process the list of channels to: # 1) make a list of channels without programme schedule data on tvg website # 2) remove these channels from their configuration list if (defined $opt->{makeignorelist}) { makeignorelist($channels); } if (defined $opt->{useignorelist}) { useignorelist($channels); warning "Remaining ".(scalar keys %$channels)." channels \n"; } #debug Dumper $channels; #------------------------------------------------------------------------------------------- # Format & write out the config file # Notifying the user :) #$bar = new XMLTV::ProgressBar({ # name => "Reformatting", # count => 1 #}) unless ($opt->{quiet} || $opt->{debug}); if (scalar keys %$channels == 0) { warning "No channels found in TVGuide \n"; exit 1; } # Let XMLTV::Writer format the results as a valid xmltv file my $result; my $writer = new XMLTV::Writer(OUTPUT => \$result, encoding => 'utf-8'); $writer->start({'generator-info-name' => $generator_info_name}); # # this writes the channels sorted by 'id' but the TVG id has no relation to the actual channel number, # and makes it harder to select the channels to fetch ### $writer->write_channels($channels); # so let's write them by name foreach (sort { $channels->{$a}->{'display-name'}[0][0] cmp $channels->{$b}->{'display-name'}[0][0] } keys %$channels) { $writer->write_channel($channels->{$_}); } # $writer->end(); #$bar->update() if defined $bar; $bar->finish() && undef $bar if defined $bar; return $result; } sub makeignorelist ($) { my $channels = shift; if (defined $opt->{makeignorelist}) { #-------------------------------------------------------------------------------------------# # this fetch pulls in a lot of 'duplicate' channels (16x Sky Sports Golf !) which, when you look at them # are devoid of listings. # let's see if we can root those out at this stage and not present them to the poor wee user (who doesn't know # which one to pick during '--configure' ) # get the listings page for this channel and see if it has data # this reduces to 1x Sky Sports Golf :-) # # reduced total to 494 channels (from 1207 in method 1) # # doing this for each channel will, of course, slow down the channel fetch # - so we only do it when asked with --makeignorelist # #-------------------------------------------------------------------------------------------# warning "Checking channels for listings data"; warning "Creating the check list will take about an hour.\n You don't need to do this every time - unless your check list \n is very out-of-date you can simply use your existing check list. \n (Ctrl+C to abort)"; my $bar = new XMLTV::ProgressBar({ name => "Checking channels", count => (scalar keys %$channels) }) unless ($opt->{quiet} || $opt->{debug}); my $skipchannels = []; my $i = 0 if $opt->{test}; my @channels = keys %$channels; for my $channel (@channels) { last if $opt->{test} and (++$i >= $opt->{test}); # limit during testing (my $id) = $channels->{$channel}->{id} =~ /^(\d+)/; next if !$id; #debug "Checking: $id"; # are there any listings for tomorrow, for this channel? my $url = $ROOT_URL.'channellistings.asp' . '?ch=' . $id . '&cTime=' . uri_escape( DateTime->today->add(days => 1)->set_time_zone('Europe/London')->strftime('%m/%d/%Y 00:00:00') ); #debug "Fetching: $url"; my $tree = fetch_url($url); if ($tree) { # (see fetch_listings for details) my @shows = $tree->look_down('_tag' => 'table', 'border' => '0', 'cellpadding' => '0', 'style' => qr/background:\s*black;border-collapse:\s*collapse;/); if (scalar @shows == 0) { # empty listings schedule push @$skipchannels, $id; debug "Channel ".$channels->{$channel}->{id}." - SKIPPED"; } } $bar->update() if defined $bar; } $bar->finish() && undef $bar if defined $bar; # write the ignore list my $fn = ($opt->{makeignorelist} || 'uktvguideignorelist'); debug "Writing $fn"; open my $fh, '>', $fn or die "Can't open file $!"; print $fh '# LIST OF IGNORED CHANNELS'."\n"; print $fh '# These channels contain no data, so it is safe to '."\n"; print $fh '# remove them from the configuration file.'."\n"; foreach (@$skipchannels) { print $fh 'channel='.$_."\n"; } close $fh; } } sub useignorelist ($) { my $channels = shift; if (defined $opt->{useignorelist}) { warning "Removing ignored channels"; # read the ignore list my $fn = ($opt->{useignorelist} || 'uktvguideignorelist'); debug "Reading $fn"; open my $fh, '<', $fn or die "Can't open file $!"; while (my $r = <$fh>) { chomp $r; next if $r =~ /^\s*#/; (my $id) = $r =~ /^channel=(\d+)$/; next if !$id; # remove from channellist delete $channels->{$id} if ($channels->{$id}); } close $fh; } } sub config_stage { my( $stage, $conf ) = @_; die "Unknown stage $stage" if $stage ne "start"; my $result; my $writer = new XMLTV::Configure::Writer( OUTPUT => \$result, encoding => 'utf-8' ); $writer->start( { grabber => $GRABBER_NAME } ); $writer->write_string({ id => 'cachedir', title => [ [ 'Directory to store the cache in', 'en' ] ], description => [ [ $GRABBER_NAME.' uses a cache with files that it has already '. 'downloaded. Please specify where the cache shall be stored. ', 'en' ] ], default => get_default_cachedir(), }); $writer->end( 'select-channels' ); return $result; } sub config_check { if (not defined( $conf->{cachedir} )) { print STDERR "No cachedir defined in configfile " . $opt->{'config-file'} . "\n" . "Please run the grabber with --configure.\n"; exit 1; } if (not defined( $conf->{'channel'} )) { print STDERR "No channels selected in configfile " . $opt->{'config-file'} . "\n" . "Please run the grabber with --configure.\n"; exit 1; } } sub fetch_url ($;$$) { # fetch a url with up to 5 retries my ($url, $method, $varhash) = @_; $XMLTV::Get_nice::FailOnError = 0; my $content; my $maxretry = 5; my $retry = 0; if (defined $method && lc($method) eq 'post') { my $ua = initialise_ua(); while ( (not defined($content)) || (length($content) == 0) ) { my $r = $ua->post($url, $varhash); $content = $r->content; if ( $r->is_error || (length($content) == 0) ) { print STDERR "HTTP error: ".$r->status_line."\n"; $retry++; return undef if $retry > $maxretry; print STDERR "Retrying URL: $url (attempt $retry of $maxretry) \n"; } } } else { while ( (not defined($content = XMLTV::Get_nice::get_nice($url))) || (length($content) == 0) ) { my $r = $XMLTV::Get_nice::Response; print STDERR "HTTP error: ".$r->status_line."\n"; $retry++; return undef if $retry > $maxretry; print STDERR "Retrying URL: $url (attempt $retry of $maxretry) \n"; } } $content = decode('UTF-8', $content); my $t = HTML::TreeBuilder->new(); $t->parse($content) or die "cannot parse content of $url\n"; $t->eof; return $t; } sub get_default_dir { my $winhome = $ENV{HOMEDRIVE} . $ENV{HOMEPATH} if defined( $ENV{HOMEDRIVE} ) and defined( $ENV{HOMEPATH} ); my $home = $ENV{HOME} || $winhome || "."; return $home; } sub get_default_cachedir { return get_default_dir() . "/.xmltv/cache"; } sub init_cachedir { my( $path ) = @_; if( not -d $path ) { mkpath( $path ) or die "Failed to create cache-directory $path: $@"; } } sub debug ( $$ ) { my( $message, $nonewline ) = @_; print STDERR $message if $opt->{debug}; print STDERR "\n" if $opt->{debug} && (!defined $nonewline || $nonewline != 1); } sub warning ( $ ) { my( $message ) = @_; print STDERR $message . "\n"; $warnings++; } sub initialise_ua { my $cookies = HTTP::Cookies->new; #my $ua = LWP::UserAgent->new(keep_alive => 1); my $ua = LWP::UserAgent->new; # Cookies $ua->cookie_jar($cookies); # Define user agent type $ua->agent('Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US)'); # Define timouts $ua->timeout(240); # Use proxy if set in http_proxy etc. $ua->env_proxy; return $ua; } # ############################################################################# __END__ =pod =head1 NAME B - Grab TV listings for UK from the TVGuide website. =head1 SYNOPSIS tv_grab_uk_tvguide --usage tv_grab_uk_tvguide --version tv_grab_uk_tvguide --configure [--config-file FILE] [--gui OPTION] [--method N] [--makeignorelist FILE] [--useignorelist FILE] [--debug] tv_grab_uk_tvguide [--config-file FILE] [--output FILE] [--days N] [--offset N] [--nodetailspage] [--legacychannels] [--quiet] [--debug] tv_grab_uk_tvguide --list-channels [--output FILE] [--method N] [--debug] =head1 DESCRIPTION Output TV listings in XMLTV format for many channels available in UK. The data come from tvguide.co.uk First you must run B to choose which channels you want to receive. Then running B with no arguments will get programme listings in XML format for the channels you chose, for available days including today. =head1 OPTIONS B<--configure> Prompt for which channels to fetch the schedule for, and write the configuration file. B<--config-file FILE> Set the name of the configuration file, the default is B<~/.xmltv/tv_grab_uk_tvguide.conf>. This is the file written by --configure and read when grabbing. B<--gui OPTION> Use this option to enable a graphical interface to be used. OPTION may be 'Tk'. Additional allowed values of OPTION are 'Term' for normal terminal output (default) and 'TermNoProgressBar' to disable the use of Term::ProgressBar. (May not work on Windows OS.) B<--days N> Grab N days. The default is 5 days. B<--offset N> Start N days in the future. The default is to start from today. B<--nodetailspage> Only fetch summary information for each programme. (See discussion below). B<--legacychannels> Channel ids were made compliant with the XMLTV specification in December 2020. Use --legacychannels to output channel ids in the previous format (i.e. number only). B<--output FILE> Write to FILE rather than standard output. B<--method N> This program has three methods for fetching the list of channels available. The preferred method can be overridden with an option of either --method 1 or --method 2 to use one of the two alternative methods. If no --method parameter is supplied then the various methods will be tried in sequence until a channels list is obtained. A parameter of --method 0 will run the preferred method only. Normally you should omit this parameter. B<--makeignorelist FILE> The source channel list contains many channels which have no programme listings available. By using a local file of channel ids these can be filtered out during the configure process. This option creates a list of channel ids which can be excluded from your config file makeignorelist can take over an hour to run but you only need run it infrequently (not every time you run --configure). B<--useignorelist FILE> Specify the file containing a list of channel ids to be removed during the configure stage. This may be the file just created with --makeignorelist (e.g. --configure --makeignorelist myfile --useignorelist myfile ) B<--quiet> Suppress the progress messages normally written to the console. B<--debug> Provide more information on progress to standard error to help in debugging. B<--list-channels> Output a list (in xmltv format) of all channels that can be fetched. B<--version> Show the version of the grabber. B<--usage> Print a help message and exit. =head1 INSTALLATION The file F has two purposes. Firstly you can map the channel ids used by the site into something more meaningful to your PVR. E.g. map==74==BBC 1 will change "74" to "BBC 1" in the output XML. Note: the lines are of the form "map=={channel id}=={my name}". The second purpose is to likewise translate genre names. So if your PVR doesn"t have a category for "Science Fiction" but uses "Sci-fi" instead, then you can specify cat==Science Fiction==Sci-fi and the output XML will have "Sci-fi". IMPORTANT: the downloaded "tv_grab_uk_tvguide.map.conf" contains example lines to illustrate the format - you should edit this file to suit your own purposes! =head1 ERROR HANDLING If the grabber fails to download data for some channel on a specific day, it will print an errormessage to STDERR and then continue with the other channels and days. The grabber will exit with a status code of 1 to indicate that the data is incomplete. =head1 ENVIRONMENT VARIABLES The environment variable HOME can be set to change where configuration files are stored. All configuration is stored in $HOME/.xmltv/. On Windows, it might be necessary to set HOME to a path without spaces in it. =head1 SUPPORTED CHANNELS For information on supported channels, see http://tvguide.co.uk/ =head1 XMLTV VALIDATION B may report an error similar to: "Line 5 Invalid channel-id BBC 1" This is a because ValidateFile.pm insists the channel-id adheres to RFC2838 despite the xmltv.dtd only saying "preferably" not "SHOULD". (Having channel ids of the form "bbc1.bbc.co.uk" will be rejected by many PVRs since they require the data to match their own list.) It may also report: "tv_sort failed on the concatenated data. Probably due to overlapping data between days." Both these errors can be ignored. =head1 USING --nodetailspage This option may be useful if you have problems accessing the tvguide website: it will considerably speed up your grabbing, but at the expense of data richness. The details page has a better list of actors, as well as director's names, film classifications, and background images. More significantly, the details page includes programme duration and programme end time. If you don't include the details page then your output programmes will not have stop times. Although stop times are an optional XMLTV element, many downstream programs expect them and break without them. Fortunately, these can be added by piping your xml file through the tv_sort filter: this will add stop times to all programmes except for the last programme on every channel. For example: tv_grab_uk_tvguide --days 3 --nodetailspage | tv_sort --by-channel --output myprogrammes.xml B As at January 2022 it seems the actor/director names and film classification is no longer present in the website, although it may be there for some channels(?). Therefore you may find the --nodetailspage option useful to significantly reduce your run time. B, the details page does include background image(s) of the programme, which is not present in your xml file when using --nodetailspage =head1 BE KIND If using the details page from the website then your grabbing might benefit from a more targeted strategy, rather than blithely getting all days for all channels. Since the published schedule rarely changes, a strategy of grabbing the next 3 days plus the 1 newest day would give you any new programmes as well as any last minute changes. Simply fetch "--offset 0 --days 3" and concatenate it with a separate fetch of "--offset 7 --days 1". For example: =over 6 ( tv_grab_uk_tvguide --days 3 --offset 0 --output temp.xml ) && ( tv_grab_uk_tvguide --days 1 --offset 7 | tv_cat --output myprogrammes.xml temp.xml - ) && ( rm -f temp.xml ) =back A similar strategy could be employed when using --nodetailspage if you have a lot of channels: =over 6 ( tv_grab_uk_tvguide --days 3 --offset 0 --nodetailspage | tv_sort --by-channel >temp.xml ) && ( tv_grab_uk_tvguide --days 1 --offset 7 --nodetailspage | tv_sort --by-channel | tv_cat --output myprogrammes.xml temp.xml - ) && ( rm -f temp.xml ) =back This avoids overloading the TVGuide website, and significantly reduces your runtime with minimal impact on your viewing schedule. TVGuide provide this data for free, so let's not abuse their generosity. =head1 DISCLAIMER The TVGuide website's license for these data does not allow non-personal use. Certainly, any commercial use of listings data obtained by using this grabber will breach copyright law, but if you are just using the data for your own personal use then you are probably fine. By using this grabber you aver you are using the listings data for your own personal use only and you absolve the author(s) from any liability under copyright law or otherwise. =head1 AUTHOR Geoff Westcott. This documentation and parts of the code based on various other tv_grabbers from the XMLTV-project. =head1 SEE ALSO L. =cut To Do ===== 1. Improve the progress bar update frequency 2. Add actor 'character' attribute DONE 30/6/16 3. Currently only does Actor, Director, Producer, Writer - does anyone actually use any of the others present in the DTD?