From 00f7c03c9ad2e9dddf92f679b0f59c8d6b47a37a Mon Sep 17 00:00:00 2001 From: David Phillips Date: Mon, 7 May 2018 18:06:17 +1200 Subject: Replace HTML::HeadParser with HTML::Parser Weird bugs with HeadParser, cannot debug and patch for upstream as yet --- Plugin/URL_Title.pm | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) (limited to 'Plugin/URL_Title.pm') diff --git a/Plugin/URL_Title.pm b/Plugin/URL_Title.pm index 8248560..5774528 100644 --- a/Plugin/URL_Title.pm +++ b/Plugin/URL_Title.pm @@ -5,7 +5,7 @@ package Plugin::URL_Title; use strict; use warnings; use HTTP::Tiny; -use HTML::HeadParser; +use HTML::Parser; use utf8; my %config; @@ -18,6 +18,17 @@ sub configure { return $self; } +my $title; + +sub start_handler +{ + return if shift ne "title"; + my $self = shift; + $self->handler(text => sub { $title = shift; }, "dtext"); + $self->handler(end => sub { shift->eof if shift eq "title"; }, + "tagname,self"); +} + sub message { my ($self, $logger, $me, $who, $where, $raw_what, $what, $irc) = @_; @@ -47,11 +58,12 @@ sub message my $html = $response->{content}; utf8::decode($html); - my $parser = HTML::HeadParser->new; - $parser->parse($html); + $title = ""; + my $p = HTML::Parser->new(api_version => 3); + $p->handler( start => \&start_handler, "tagname,self"); + $p->parse($html); + die "Error: $!\n" if $!; - # get title and unpack from utf8 (assumption) - my $title = $parser->header("title"); utf8::upgrade($title); return unless $title; -- cgit v1.1 From 817ebdeb8eeaa344fd77a2c84d4e90d4cdf5f66a Mon Sep 17 00:00:00 2001 From: David Phillips Date: Mon, 7 May 2018 20:57:27 +1200 Subject: Fold URL title whitespace into same line --- Plugin/URL_Title.pm | 3 +++ 1 file changed, 3 insertions(+) (limited to 'Plugin/URL_Title.pm') diff --git a/Plugin/URL_Title.pm b/Plugin/URL_Title.pm index 5774528..1d0056e 100644 --- a/Plugin/URL_Title.pm +++ b/Plugin/URL_Title.pm @@ -64,6 +64,9 @@ sub message $p->parse($html); die "Error: $!\n" if $!; + $title =~ s/\s+/ /g; + $title =~ s/(^\s+|\s+$)//g; + utf8::upgrade($title); return unless $title; -- cgit v1.1 From eb2ff6434ee9ffe9354539186bbb7b8726788e0f Mon Sep 17 00:00:00 2001 From: David Phillips Date: Mon, 7 May 2018 21:17:47 +1200 Subject: Truncate URLs based on shorturl length, not full URL --- Plugin/URL_Title.pm | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'Plugin/URL_Title.pm') diff --git a/Plugin/URL_Title.pm b/Plugin/URL_Title.pm index 1d0056e..3297bed 100644 --- a/Plugin/URL_Title.pm +++ b/Plugin/URL_Title.pm @@ -71,12 +71,13 @@ sub message return unless $title; my $shorturl = $url; - $shorturl = (substr $url, 0, $config{url_len}) . "…" if length ($url) > $config{url_len}; - # remove http(s):// to avoid triggering other poorly configured bots $shorturl =~ s,^https?://,,g; $shorturl =~ s,/$,,g; + # truncate URL without http(s):// to configured length if needed + $shorturl = (substr $shorturl, 0, $config{url_len}) . "…" if length ($shorturl) > $config{url_len}; + my $composed_title = "$title ($shorturl)"; return $composed_title; } -- cgit v1.1 From 53718f690df53285362145ee474f40a2f5cc63e7 Mon Sep 17 00:00:00 2001 From: David Phillips Date: Tue, 19 Jun 2018 00:12:14 +1200 Subject: URL_Title: Allow SVG titling --- Plugin/URL_Title.pm | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'Plugin/URL_Title.pm') diff --git a/Plugin/URL_Title.pm b/Plugin/URL_Title.pm index 3297bed..6d46e43 100644 --- a/Plugin/URL_Title.pm +++ b/Plugin/URL_Title.pm @@ -41,7 +41,9 @@ sub message } return unless $url; - my $http = HTTP::Tiny->new((default_headers => {'Range' => "bytes=0-65536", 'Accept' => 'text/html'}, timeout => 3)); + # FIXME add more XML-based formats that we can theoretically extract titles from + # FIXME factor out accepted formats and response match into accepted formats array + my $http = HTTP::Tiny->new((default_headers => {'Range' => "bytes=0-65536", 'Accept' => 'text/html, image/svg+xml'}, timeout => 3)); my $response = $http->get($url); @@ -50,8 +52,8 @@ sub message return; } - if (!($response->{headers}->{"content-type"} =~ m,text/html ?,)) { - $logger->("Not html, giving up now"); + if (!($response->{headers}->{"content-type"} =~ m,(text/html|image/svg\+xml),)) { + $logger->("I don't think I can parse titles from $response->{headers}->{'content-type'} - stopping here"); return; } -- cgit v1.1 From 792eade6c29beb6cf9b1e31ddffdc6476ff21b01 Mon Sep 17 00:00:00 2001 From: David Phillips Date: Wed, 20 Jun 2018 17:41:52 +1200 Subject: Change URL parsing from space to RFC 3982 --- Plugin/URL_Title.pm | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'Plugin/URL_Title.pm') diff --git a/Plugin/URL_Title.pm b/Plugin/URL_Title.pm index 6d46e43..a2c8930 100644 --- a/Plugin/URL_Title.pm +++ b/Plugin/URL_Title.pm @@ -36,7 +36,8 @@ sub message return if ($config{url_on} == 0); - if ($what =~ /(https?:\/\/[^ ]+)/i) { + # Drawn from RFC 3986§2 + if ($what =~ /(https?:\/\/[A-z0-9\-\._~:\/\?#\[\]@\!\$&'()\*\+,;=]+)/i) { $url = $1; } return unless $url; -- cgit v1.1 From 549ac4f97e0a70238006cc7b4d4ff752f281b930 Mon Sep 17 00:00:00 2001 From: David Phillips Date: Wed, 20 Jun 2018 17:49:19 +1200 Subject: No need for [A-z] when case-insensitive flag used --- Plugin/URL_Title.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Plugin/URL_Title.pm') diff --git a/Plugin/URL_Title.pm b/Plugin/URL_Title.pm index a2c8930..73fa498 100644 --- a/Plugin/URL_Title.pm +++ b/Plugin/URL_Title.pm @@ -37,7 +37,7 @@ sub message return if ($config{url_on} == 0); # Drawn from RFC 3986§2 - if ($what =~ /(https?:\/\/[A-z0-9\-\._~:\/\?#\[\]@\!\$&'()\*\+,;=]+)/i) { + if ($what =~ /(https?:\/\/[a-z0-9\-\._~:\/\?#\[\]@\!\$&'()\*\+,;=]+)/i) { $url = $1; } return unless $url; -- cgit v1.1