package RDF::MicroTurtle;

# This isn't ready for general release yet.

use utf8;
use strict;
use Digest::SHA1;
use Encode qw(_utf8_on);
use LWP::Simple;
use RDF::Redland;
use Text::Balanced qw(extract_bracketed extract_delimited);
use URI;

BEGIN
{
	our $count = 0;
	
	# Certain prefixes are hard-coded. This list is part of the MicroTurtle spec.
	# (Or it will be if a spec gets written.) The list has a number of good,
	# general-purpose vocabs, plus a number which have been chosen because they
	# seem good matches for the type of topics frequently discussed in microblogs.
	#
	# For other prefixes, prefix.cc's service is used, but that is not a very
	# stable way of doing things.
	our $expand = {
		'bio'     => 'http://purl.org/vocab/bio/0.1/' ,
		'cc'      => 'http://creativecommons.org/ns#' ,
		'ccold'   => 'http://web.resource.org/cc/' ,
		'ccrel'   => 'http://creativecommons.org/ns#' ,
		'dc'      => 'http://purl.org/dc/terms/' ,
		'dc11'    => 'http://purl.org/dc/elements/1.1/' ,
		'dcterms' => 'http://purl.org/dc/terms/' ,
		'doac'    => 'http://ramonantonio.net/doac/0.1/#' ,
		'doap'    => 'http://usefulinc.com/ns/doap#' ,
		'foaf'    => 'http://xmlns.com/foaf/0.1/' ,
		'geo'     => 'http://www.w3.org/2003/01/geo/wgs84_pos#' , 
		'ical'    => 'http://www.w3.org/2002/12/cal/icaltzd#' ,
		'lac'     => 'http://laconi.ca/ont/' ,
		'like'    => 'http://ontologi.es/like#' ,
		'log'     => 'http://www.w3.org/2000/10/swap/log#' ,
		'mo'      => 'http://purl.org/ontology/mo/' ,
		'ov'      => 'http://open.vocab.org/terms/' ,
		'owl'     => 'http://www.w3.org/2002/07/owl#' ,
		'rdf'     => 'http://www.w3.org/1999/02/22-rdf-syntax-ns#' ,
		'rdfg'    => 'http://www.w3.org/2004/03/trix/rdfg-1/' ,
		'rdfs'    => 'http://www.w3.org/2000/01/rdf-schema#' ,
		'rel'     => 'http://purl.org/vocab/relationship/' ,
		'rev'     => 'http://purl.org/stuff/rev#' ,
		'rss'     => 'http://purl.org/rss/1.0/' ,
		'sioc'    => 'http://rdfs.org/sioc/ns#' ,
		'skos'    => 'http://www.w3.org/2004/02/skos/core#' ,
		'tags'    => 'http://www.holygoat.co.uk/owl/redwood/0.1/tags/' ,
		'vcard'   => 'http://www.w3.org/2006/vcard/ns#' ,
		'wot'     => 'http://xmlns.com/wot/0.1/' ,
		'xfn'     => 'http://vocab.sindice.com/xfn#' ,
		'xhv'     => 'http://www.w3.org/1999/xhtml/vocab#' ,
		'xsd'     => 'http://www.w3.org/2001/XMLSchema#' ,
		};
}

sub new
{
	my $class = shift;
	my %arg   = @_;
	my $self  = {};
		
	bless $self, $class;
	
	die "Must provide 'text' to this constructor."
		unless (defined $arg{'text'});
	die "Must provide 'graph_uri' to this constructor."
		unless (defined $arg{'graph_uri'});
	die "Must provide 'message_uri' to this constructor."
		unless (defined $arg{'message_uri'});
	
	$self->{'default_graph'} = $arg{'graph_uri'};

	$self->{'arguments'} = \%arg;

	return unless $arg{'text'} =~ /^(.*)#m?ttl(.+)$/;
	my ($comment , $body) = ($1, $2);
	
	my $g = $self->graph;
	
	$self->{'arguments'}->{'me_uri'}      ||= $arg{'message_uri'} . '#me';
	$self->{'arguments'}->{'person_uri'}  ||= $arg{'message_uri'} . '#person/%s/';
	$self->{'arguments'}->{'person_page'} ||= $arg{'message_uri'} . '#person/%s/page';
	$self->{'arguments'}->{'tag_uri'}     ||= 'http://tags.moat-project.org/tag/%s';
	$self->{'arguments'}->{'tag_page'}    ||= 'http://tags.moat-project.org/tag/%s.html';
	
	# Graph
	$g->add(
		RDF::Redland::URI->new( $arg{'graph_uri'} ),
		RDF::Redland::URI->new('http://www.w3.org/1999/02/22-rdf-syntax-ns#type'),
		RDF::Redland::URI->new('http://www.w3.org/2004/03/trix/rdfg-1/Graph'));
	# ... comment
	$g->add(
		RDF::Redland::URI->new( $arg{'graph_uri'} ),
		RDF::Redland::URI->new('http://www.w3.org/2000/01/rdf-schema#comment'),
		RDF::Redland::LiteralNode->new($comment))
		if length $comment;
	
	&ttl_parser($self, $g, $arg{'message_uri'}, $body, {
		'me'       => $self->{'arguments'}->{'me_uri'} ,
		'user'     => $self->{'arguments'}->{'person_uri'} ,
		'userfunc' => $self->{'arguments'}->{'person_uri_function'} ,
		'userpage' => $self->{'arguments'}->{'person_page'} ,
		'tag'      => $self->{'arguments'}->{'tag_uri'} ,
		'tagpage'  => $self->{'arguments'}->{'tag_page'}
		});

	return $self;
}

sub new_from_feed
{
	my $class = shift;
	my %arg   = @_;
	my $self  = {};
		
	bless $self, $class;
	
	if (length $arg{'identica_account'})
	{
		$arg{'feed_uri'}    ||= $arg{'identica_account'} . "/rss";
		$arg{'me_uri'}      ||= $arg{'identica_account'} . "#me";
		$arg{'person_uri'}  ||= 'http://identi.ca/%s#me';
		$arg{'person_page'} ||= 'http://identi.ca/%s';
		$arg{'tag_uri'}     ||= 'http://tags.moat-project.org/tag/%s';
		$arg{'tag_page'}    ||= 'http://identi.ca/tag/%s';
	}
	
	$self->{'arguments'} = \%arg;
	
	my $rss;
	if ($arg{'feed_file'})
	{
		local $/;
		open RSS, "<:utf8", $arg{'feed_file'};
		$rss = <RSS>;
		close RSS;
	}
	else
	{
		$rss = &get( $arg{feed_uri} );
		_utf8_on($rss);
	}
	
	# Fix silly laconica RSS bug
	$rss =~ s#<sioct:MicroblogPost rdf:resource="(http.+)"></sioct:MicroblogPost>#<rdf:li rdf:resource="\1" />#g
		if ($rss =~ m?xmlns:laconica="http://laconi.ca/ont/"?);
	$rss =~ s#\\foo##;

	$self->{'feed'}          = $rss;
	$self->{'default_graph'} = $self->{'arguments'}->{'feed_uri'};

	my $parser  = RDF::Redland::Parser->new("rdfxml");
	$parser->parse_string_into_model(
		$self->{'feed'},
		RDF::Redland::URI->new($self->{'arguments'}->{'feed_uri'}),
		$self->graph($self->{'arguments'}->{'feed_uri'}));
	
	$self->process_feed_model
		unless $arg{'manual_mode'};

	return $self;
}

# Given a URL, returns Redland model of a graph such that <url> log:semantics <graph> .
sub graph
{
	my $self   = shift;
	my $graph  = shift;

	$graph = $self->{'default_graph'}
		if (!$graph && $self->{'default_graph'});

	die "No graph provided\n"
		if !$graph;

	my $s = RDF::Redland::Storage->new("hashes", "test", "new='yes',hash-type='memory'");
	$self->{'graphs'}->{$graph} = RDF::Redland::Model->new($s, '')
		unless $self->{'graphs'}->{$graph};

	return $self->{'graphs'}->{$graph};
}

# Given a URL, returns BNode identifier of a graph such that <url> log:semantics <graph> .
sub graph_id
{
	my $self   = shift;
	my $graph  = shift;

	return 'g'.Digest::SHA1::sha1_hex($graph);
}

sub process_feed_model
{
	my $self  = shift;
	my $major = $self->graph($self->{'arguments'}->{'feed_uri'});
	
	# Query for RSS items
	my $query   = 'SELECT * '
		    . 'WHERE '
		    . '{ '
		    .      '?resource a <http://purl.org/rss/1.0/item> ; '
		    .                '<http://purl.org/rss/1.0/title> ?title ; '
		    .                '<http://purl.org/rss/1.0/link> ?link . '
		    . '}';
	my $results = $major->query_execute(
		RDF::Redland::Query->new(
			$query,
			RDF::Redland::URI->new($self->{'arguments'}->{'feed_uri'}),
			undef,
			'sparql'
			)
		);

	# Loop results
	while(!$results->finished)
	{
		my %r = $results->bindings;
		$results->next_result;

		my $title = $r{'title'}->literal_value;
		my $link  = ('RDF::Redland::URI' eq ref $r{'link'})
			  ? $r{'link'}->uri->as_string
			  : $r{'link'}->literal_value;
		
		_utf8_on($title);
		_utf8_on($link);

		next unless $title =~ /^(.*)#m?ttl(.+)$/;
		
		my ($comment , $body) = ($1, $2);
		$comment =~ s/\s+$//;
		
		my $minor = $self->graph($link);

		# Copy relevent information from major graph to minor.
		my $furtherInfoQuery   = 'SELECT * '
			    . 'WHERE '
			    . '{ '
			    .  ' <' . $r{'resource'}->uri->as_string . '> ?p ?o . '
			    . '}';
		my $furtherInfoResults = $major->query_execute(
			RDF::Redland::Query->new(
				$furtherInfoQuery,
				RDF::Redland::URI->new($self->{'arguments'}->{'feed_uri'}),
				undef,
				'sparql'
				)
			);
		while(!$furtherInfoResults->finished)
		{
			my %r2 = $furtherInfoResults->bindings;
			$furtherInfoResults->next_result;
			next unless $r2{'p'}->uri->as_string =~ m!^(
					http://purl.org/dc/ |
					http://purl.org/rss/1.0/ | 
					http://www.w3.org/1999/02/22-rdf-syntax-ns#
				)!ix;

			$minor->add(
				$r{'resource'},
				$r2{'p'},
				$r2{'o'});
		}
		
		# Graph
		$minor->add(
			RDF::Redland::BlankNode->new( $self->graph_id($link) ),
			RDF::Redland::URI->new('http://www.w3.org/1999/02/22-rdf-syntax-ns#type'),
			RDF::Redland::URI->new('http://www.w3.org/2004/03/trix/rdfg-1/Graph'));
		# ... is semantics of link
		$minor->add(
			RDF::Redland::URI->new($link),
			RDF::Redland::URI->new('http://www.w3.org/2000/10/swap/log#semantics'),
			RDF::Redland::BlankNode->new( $self->graph_id($link) ));
		# ... comment
		$minor->add(
			RDF::Redland::BlankNode->new( $self->graph_id($link) ),
			RDF::Redland::URI->new('http://www.w3.org/2000/01/rdf-schema#comment'),
			RDF::Redland::LiteralNode->new($comment))
			if length $comment;
		
		&ttl_parser($self, $minor, $link, $body, {
			'me'       => $self->{'arguments'}->{'me_uri'} ,
			'user'     => $self->{'arguments'}->{'person_uri'} ,
			'userfunc' => $self->{'arguments'}->{'person_uri_function'} ,
			'userpage' => $self->{'arguments'}->{'person_page'} ,
			'tag'      => $self->{'arguments'}->{'tag_uri'} ,
			'tagpage'  => $self->{'arguments'}->{'tag_page'}
			});
		
		# Also add graph to major model
		$major->add(
			RDF::Redland::BlankNode->new( $self->graph_id($link) ),
			RDF::Redland::URI->new('http://www.w3.org/1999/02/22-rdf-syntax-ns#type'),
			RDF::Redland::URI->new('http://www.w3.org/2004/03/trix/rdfg-1/Graph'));
		$major->add(
			RDF::Redland::URI->new($link),
			RDF::Redland::URI->new('http://www.w3.org/2000/10/swap/log#semantics'),
			RDF::Redland::BlankNode->new( $self->graph_id($link) ));
		
		# And add reified statements
		my $stats = $minor->as_stream;
		while ($stats && !$stats->end)
		{
			my $s = $stats->current;			
			my $statement = RDF::Redland::BlankNode->new(&_ttl_random_bnode);
			
			$major->add(
				RDF::Redland::BlankNode->new( $self->graph_id($link) ),
				RDF::Redland::URI->new('http://open.vocab.org/terms/statement'),
				$statement);
			$major->add(
				$statement,
				RDF::Redland::URI->new('http://www.w3.org/1999/02/22-rdf-syntax-ns#type'),
				RDF::Redland::URI->new('http://www.w3.org/1999/02/22-rdf-syntax-ns#Statement'));
			$major->add(
				$statement,
				RDF::Redland::URI->new('http://www.w3.org/1999/02/22-rdf-syntax-ns#subject'),
				$s->subject);
			$major->add(
				$statement,
				RDF::Redland::URI->new('http://www.w3.org/1999/02/22-rdf-syntax-ns#predicate'),
				$s->predicate);
			$major->add(
				$statement,
				RDF::Redland::URI->new('http://www.w3.org/1999/02/22-rdf-syntax-ns#object'),
				$s->object);
			
			$stats->next;
		}
	}
}

sub to_string
{
	my $self  = shift;
	my $fmt   = shift;
	my $graph = shift;
	
	my $model = $graph;
	$model = $self->graph($graph)
		unless UNIVERSAL::isa($model, 'RDF::Redland::Model');
	
	my $rv;
	
	die "No suitable model.\n"
		unless ($model);
	
	my $serializer = RDF::Redland::Serializer->new($fmt);
	foreach my $k (keys %{$RDF::MicroTurtle::expand})
	{
		next if $k eq '_';
		$serializer->set_namespace($k, RDF::Redland::URI->new($RDF::MicroTurtle::expand->{$k}));
	}
	$rv = $serializer->serialize_model_to_string(undef, $model);
	
	# Improve turtle readability.
	if ($fmt eq 'turtle')
	{
		my @bits = split /\n\n/, $rv;
		my $prefixes = shift @bits;
		my $body = join "\n\n", @bits;
		$rv = '';
		
		@bits = split /\n/, $prefixes;
		$prefixes = [];
		foreach my $line (@bits)
		{
			if ($line =~ /\@prefix \s* ([^\s\:]+) \s* \: \s* <([^>]+)> \s* \./x)
			{
				push @$prefixes, { 'prefix'=>$1 , 'uri'=>$2 };
			}
			else
			{
				$rv .= "$line\n";
			}
		}
		
		my @sorted = sort { $a->{'prefix'} cmp $b->{'prefix'} } @$prefixes;
		foreach my $p (@sorted)
		{
			$rv .= sprintf("\@prefix %-8s : <%s> .\n", $p->{'prefix'}, $p->{'uri'})
				if $body =~ /$p->{prefix}:/;
		}
		
		$rv .= "\n$body";
	}
	
	# Improve ntriples readability.
	elsif ($fmt eq 'ntriples')
	{
		my @lines  = split /\n/, $rv;
		my @sorted = sort @lines;
		$rv = join "\n", @sorted;
	}

	return $rv;
}

sub ttl_parser
{
	my ($self, $graph, $res, $ttl, $opts) = @_;
	
	my $parsed  = _ttl_parse($ttl, $res, $opts);
	my $noded   = _ttl_nodify($parsed->{'triples'}, $res);

	foreach my $t (@{$noded->{'triples'}})
	{
		$graph->add(@$t);
	}
	
	foreach my $subj (keys %{$parsed->{'tags'}})
	{
		foreach my $tag (@{$parsed->{'tags'}->{$subj}})
		{
			$graph->add(	
				$noded->{'mapping'}->{$subj},
				RDF::Redland::URI->new('http://www.holygoat.co.uk/owl/redwood/0.1/tags/taggedWithTag'),
				RDF::Redland::URI->new(sprintf($opts->{tag}, $tag)))
				if defined $noded->{'mapping'}->{$subj};
			$graph->add(	
				RDF::Redland::URI->new(sprintf($opts->{tag}, $tag)),
				RDF::Redland::URI->new('http://xmlns.com/foaf/0.1/homepage'),
				RDF::Redland::URI->new(sprintf($opts->{tagpage}, $tag)));
			$graph->add(	
				RDF::Redland::URI->new(sprintf($opts->{tag}, $tag)),
				RDF::Redland::URI->new('http://www.holygoat.co.uk/owl/redwood/0.1/tags/name'),
				RDF::Redland::LiteralNode->new($tag));
		}
	}
}

sub _ttl_nodify
{
	my $triples = shift;
	my $res     = shift;
	my $mapping = { "<$res>" => RDF::Redland::URINode->new($res) };
	my $rv = [];
	
	foreach my $t (@$triples)
	{
		my @nodes = ();
		foreach my $n (@$t)
		{
			if ((ref $n) && $n->isa('RDF::MicroTurtle::Literal'))
			{
				push @nodes,
					RDF::Redland::LiteralNode->new(
						$n->{'value'},
						RDF::Redland::URI->new($n->{'dturi'}),
						$n->{'lang'});
			}
			elsif ($n =~ /^<(.*)>$/)
			{
				$mapping->{$n} = RDF::Redland::URINode->new($1)
					if (!defined $mapping->{$n});
				push @nodes, $mapping->{$n};
			}
			elsif ($n =~ /^_:(.+)$/)
			{
				$mapping->{$n} = RDF::Redland::BlankNode->new($1)
					if (!defined $mapping->{$n});
				push @nodes, $mapping->{$n};
			}
			elsif ($n =~ /^_:$/)
			{
				$mapping->{'_:'} = RDF::Redland::BlankNode->new('ARGH')
					if (!defined $mapping->{'_:'});
				push @nodes, $mapping->{'_:'};
			}
			elsif ($n =~ /^"(.*)"$/)
			{
				push @nodes, RDF::Redland::LiteralNode->new($1);
			}
		}
		
		push @$rv, \@nodes;
	}
	
	return {
		'triples' => $rv ,
		'mapping' => $mapping ,
		};
}

sub _ttl_expand
{
	my $base  = shift;
	my $opts  = shift;
	my $uri   = URI->new($base);
	my @rv;
	
	while (my $term = shift @_)
	{
		if ((ref $term) && $term->isa('RDF::MicroTurtle::Literal'))
		{
			push @rv, $term;
		}
		elsif ($term =~ /^<\@(.+)>$/)
		{
			if ((defined $opts->{userfunc}) && ('CODE' eq ref $opts->{userfunc}))
			{
				my $f = $opts->{userfunc};
				push @rv, &$f($1);
			}
			else
			{
				push @rv, sprintf("<$opts->{user}>", $1);
			}
		}
		elsif ($term =~ /^<#me>$/)
		{
			push @rv, sprintf("<%s>", $opts->{'me'});
		}
		elsif ($term =~ /^([A-Za-z0-9_-]+):(\S+)$/
		&&     $term !~ /^_:/)
		{
			my ($prefix, $suffix) = ($1, $2);
			if (!defined $RDF::MicroTurtle::expand->{$prefix})
			{
				my $txt = &get("http://prefix.cc/$prefix.txt.plain");
				chomp $txt;
				my (undef, $uri) = split /\s/, $txt;
				$RDF::MicroTurtle::expand->{$prefix} = $uri;
			}
			push @rv, '<' . $RDF::MicroTurtle::expand->{$prefix} . $suffix . '>';
		}
		elsif ($term =~ /^<(.+)>$/)
		{
			my $termuri = URI->new_abs($1, $uri);
			push @rv, "<$termuri>";
		}
		elsif ($term =~ /^<>$/)
		{
			push @rv, "<$uri>";
		}
		else
		{
			push @rv, $term;
		}
	}
	
	return @rv;
}

sub _ttl_random_bnode
{
	return sprintf('b%08d', $RDF::MicroTurtle::count++);
}

sub _ttl_parse
{
	my $template_string = shift;
	my $base            = shift;
	my $opts            = shift;
	
	my @tokens = _ttl_expand($base, $opts, _ttl_tokenize($template_string));
	my $triples;
	my $tags = {};
	my $current_triple;
	my $isof = 0;
	
	while (my $t = shift @tokens)
	{
		if ((ref $t) && $t->isa('RDF::MicroTurtle::Literal'))
		{
			push @$current_triple, $t;
		}
		elsif ($t =~ /^#(\S+)$/)
		{
			#hashtags 
			if (defined $current_triple->[0])
			{
				push @{$tags->{$current_triple->[0]}}, $1;
			}
			else
			{
				push @{$tags->{'<'.$base.'>'}}, $1;
			}
		}
		elsif ($t eq '.')
		{
			if (defined $current_triple->[2] && !defined $current_triple->[3])
			{
				$current_triple = [$current_triple->[2], $current_triple->[1], $current_triple->[0]] if $isof;
				push @$triples, $current_triple;
				$isof = 0;
			}
			$current_triple = [];
		}
		elsif ($t eq ';')
		{
			if (defined $current_triple->[2] && !defined $current_triple->[3])
			{
				$current_triple = [$current_triple->[2], $current_triple->[1], $current_triple->[0]] if $isof;
				push @$triples, $current_triple;
				$isof = 0;
			}
			$current_triple = [$current_triple->[0]];
		}
		elsif ($t eq ',')
		{
			if (defined $current_triple->[2] && !defined $current_triple->[3])
			{
				$current_triple = [$current_triple->[2], $current_triple->[1], $current_triple->[0]] if $isof;
				push @$triples, $current_triple;
			}
			$current_triple = [$current_triple->[2], $current_triple->[1], $current_triple->[0]] if $isof;
			$current_triple = [$current_triple->[0] , $current_triple->[1]];
		}
		elsif ($t eq '[]')
		{
			push @$current_triple, '_:'.&_ttl_random_bnode;
		}
		elsif ($t eq 'is')
		{
			# noop
		}
		elsif ($t eq 'of')
		{
			$isof = 1;
		}
		elsif ($t eq '[')
		{
			my $anon = '_:'.&_ttl_random_bnode;
			my $r = &_ttl_parse_bracketted(\@tokens, $anon, $triples, $tags);
			foreach my $triple (@{ $r->{'triples'} })
			{
				push @$triples, $triple;
			}
			push @$current_triple, $anon;
		}
		else
		{
			push @$current_triple, $t;
		}
	}
	
	if (defined $current_triple->[2] && !defined $current_triple->[3])
	{
		$current_triple = [$current_triple->[2], $current_triple->[1], $current_triple->[0]] if $isof;
		push @$triples, $current_triple;
		$isof = 0;
	}
	
	return {
		'triples' => $triples ,
		'tags'    => $tags
		};
}


sub _ttl_parse_bracketted
{
	my $tokens  = shift;
	my $subject = shift;
	my $triples = shift;
	my $tags    = shift;
	
	my $current_triple = [$subject];
	my $isof = 0;
	
	while (my $t = shift @$tokens)
	{
		last if (!ref $t) && ($t eq ']');
	
		if ((ref $t) && $t->isa('RDF::MicroTurtle::Literal'))
		{
			push @$current_triple, $t;
		}
		elsif ($t =~ /^#(\S+)$/)
		{
			push @{$tags->{$subject}}, $1;  #hashtags
		}
		elsif ($t eq ';')
		{
			if (defined $current_triple->[2] && !defined $current_triple->[3])
			{
				$current_triple = [$current_triple->[2], $current_triple->[1], $current_triple->[0]] if $isof;
				push @$triples, $current_triple;
				$isof = 0;
			}
			$current_triple = [$subject];
		}
		elsif ($t eq ',')
		{
			if (defined $current_triple->[2] && !defined $current_triple->[3])
			{
				$current_triple = [$current_triple->[2], $current_triple->[1], $current_triple->[0]] if $isof;
				push @$triples, $current_triple;
			}
			$current_triple = [$current_triple->[2], $current_triple->[1], $current_triple->[0]] if $isof;
			$current_triple = [$subject, $current_triple->[1]];
		}
		elsif ($t eq '[]')
		{
			push @$current_triple, '_:'.&_ttl_random_bnode;
		}
		elsif ($t eq '[')
		{
			my $anon = '_:'.&_ttl_random_bnode;
			my $r = &_ttl_parse_bracketted($tokens, $anon, $triples, $tags);
			foreach my $triple (@{ $r->{'triples'} })
			{
				push @$triples, $triple;
			}
			push @$current_triple, $anon;
		}
		else
		{
			push @$current_triple, $t;
		}
	}
	
	if (defined $current_triple->[2] && !defined $current_triple->[3])
	{
		$current_triple = [$current_triple->[2], $current_triple->[1], $current_triple->[0]] if $isof;
		push @$triples, $current_triple;
		$isof = 0;
	}
		
	return {
		'triples' => $triples ,
		'tags'    => $tags
		};
}

sub _ttl_tokenize
{
	my $ts   = shift;
	my @rv;
	
	if ($ts =~ /^[\s\r\n]*\{.*\}[\s\r\n]*$/s)
	{
		$ts =~ s/^([\s\r\n]*\{)|(\}[\s\r\n]*$)//gs;
	}
	
	while (length $ts)
	{
		$ts =~ s/^[\s\r\n]+//s;
		
		if ($ts =~ /^([\-\+]?([0-9]+\.[0-9]*e[\-\+]?[0-9]+))/i
		||     $ts =~ /^([\-\+]?(\.[0-9]+e[\-\+]?[0-9]+))/i
		||     $ts =~ /^([\-\+]?([0-9]+e[\-\+]?[0-9]+))/i)
		{
			push @rv, RDF::MicroTurtle::Literal->new($1, undef, 'xsd:double');
			$ts = substr($ts, length $1);
		}
		elsif ($ts =~ /^([\-\+]?([0-9]+\.[0-9]*))/
		||     $ts =~ /^([\-\+]?(\.[0-9]+))/)
		{
			push @rv, RDF::MicroTurtle::Literal->new($1, undef, 'xsd:decimal');
			$ts = substr($ts, length $1);
		}
		elsif ($ts =~ /^([\-\+]?([0-9]+))/)
		{
			push @rv, RDF::MicroTurtle::Literal->new($1, undef, 'xsd:integer');
			$ts = substr($ts, length $1);
		}
		elsif ($ts =~ /^([\.\,\;])/)
		{
			push @rv, $1;
			$ts = substr($ts, length $1);
		}
# Don't have placeholder variables.
#		elsif ($ts =~ /^([\?\$]\S+)/)
#		{
#			push @rv, $1;
#			$ts = substr($ts, length $1);
#		}
# But have hashtags...
		elsif ($ts =~ /^([\#][A-Za-z0-9_\.-]+)/)
		{
			push @rv, $1;
			$ts = substr($ts, length $1);
		}
		elsif ($ts =~ /^(<[^>]*>)/)
		{
			push @rv, $1;
			$ts = substr($ts, length $1);
		}
		elsif ($ts =~ m'^(https?://\S+)'i)
		{
			push @rv, '<'.$1.'>';
			$ts = substr($ts, length $1);
		}
		elsif ($ts =~ /^["']/)
		{
			my ($string, $lang, $dt, $dturi);
			($string, $ts) = extract_delimited($ts, substr($ts, 0, 1));
			
			if ($ts =~ /^(
				\@
				(
					(([A-Z]{2,8})|i|x)		# primary
					(\-[A-Z]{4})?			# script
					(\-([A-Z]{2})|([0-9]{3}))?	# region
					(\-[A-Z0-9]+)*			# other bits
				)
				)/ix)
			{
				$lang = lc $2;
				$ts = substr($ts, length $1);
			}
			if ($ts =~ /^(\^\^<([^>]*)>)/)
			{
				$dturi = $2;
				$ts = substr($ts, length $1);
			}
			elsif ($ts =~ /^(\^\^([^:]*:[^\s\,\;]*))/)
			{
				$dt = $2;
				$ts = substr($ts, length $1);
			}
			
			$string = substr($string, 1, (length $string)-2);
			
			push @rv, RDF::MicroTurtle::Literal->new($string, $lang, $dt, $dturi);
		}
		elsif ($ts =~ /^(_:[^\s\,\;]*)/)
		{
			push @rv, $1;
			$ts = substr($ts, length $1);
		}
		elsif ($ts =~ /^([^\s\,\;]*:[^\s\,\;]*)/)
		{
			push @rv, $1;
			$ts = substr($ts, length $1);
		}
		elsif ($ts =~ /^a\b/)
		{
			push @rv, '<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>';
			$ts = substr($ts, 1);
		}
		elsif ($ts =~ /^of\b/)
		{
			push @rv, 'of';
			$ts = substr($ts, 2);
		}
		elsif ($ts =~ /^is\b/)
		{
			push @rv, 'is';
			$ts = substr($ts, 2);
		}
		elsif ($ts =~ /^=>/)
		{
			push @rv, '<http://www.w3.org/2000/10/swap/log#implies>';
			$ts = substr($ts, 2);
		}
		elsif ($ts =~ /^<=/)
		{
			push @rv, ('is', '<http://www.w3.org/2000/10/swap/log#implies>', 'of');
			$ts = substr($ts, 2);
		}
		elsif ($ts =~ /^at\b/)
		{
			push @rv, '<http://www.w3.org/2003/01/geo/wgs84_pos#location>';
			$ts = substr($ts, 2);
		}
		elsif ($ts =~ /^says\b/)
		{
			push @rv, '<http://open.vocab.org/terms/quote>';
			$ts = substr($ts, 4);
		}
		elsif ($ts =~ /^gist\b/)
		{
			push @rv, '<http://purl.org/dc/terms/abstract>';
			$ts = substr($ts, 4);
		}
		elsif ($ts =~ /^=/)
		{
			push @rv, '<http://www.w3.org/2002/07/owl#sameAs>';
			$ts = substr($ts, 1);
		}
		elsif ($ts =~ /^\s*([→])/)
		{
			push @rv, '<http://xmlns.com/foaf/0.1/homepage>';
			$ts = substr($ts, length($1));
		}
		elsif ($ts =~ /^\s*([←])/)
		{
			push @rv, ('<http://xmlns.com/foaf/0.1/primaryTopic>');
			$ts = substr($ts, length($1));
		}
		elsif ($ts =~ /^\s*([♥♡❤])/)
		{
			push @rv, '<http://ontologi.es/like#likes>';
			$ts = substr($ts, length($1));
		}
		elsif ($ts =~ /^(true|false)\b/)
		{
			push @rv, RDF::MicroTurtle::Literal->new($1, undef, 'xsd:boolean');
			$ts = substr($ts, length $1);
		}
		elsif ($ts =~ /^(\@[A-Za-z0-9][A-Za-z0-9-]*)\b/)
		{
			push @rv, "<$1>";
			$ts = substr($ts, length $1);
		}
		elsif ($ts =~ /^(\[\s*\])/)
		{
			push @rv, '[]';
			$ts = substr($ts, length $1);
		}
		elsif ($ts =~ /^([\[\]\(\)])/)
		{
			push @rv, $1;
			$ts = substr($ts, 1);
		}
		else
		{
			warn "Possible Turtle tokenisation problem!\n";
			push @rv, "???ERRORCOND???";
			return @rv;
		}
	}
	
	return @rv;
}

1;

# Utility class

package RDF::MicroTurtle::Literal;

use overload '""' => sub { return $_[0]->{'value'}; } ;
use LWP::Simple;

sub new
{
	my $class = shift;
	my $value = shift;
	my $lang  = shift;
	my $dt    = shift;
	my $dturi = shift;
	
	if (!$dturi && $dt =~ /^([^:]+):(.*)$/)
	{
		my ($prefix, $suffix) = ($1, $2);
		if (!defined $RDF::MicroTurtle::expand->{$prefix})
		{
			my $txt = &get("http://prefix.cc/$prefix.txt.plain");
			chomp $txt;
			my (undef, $uri) = split /\s/, $txt;
			$RDF::MicroTurtle::expand->{$prefix} = $uri;
		}
		
		$dturi = $RDF::MicroTurtle::expand->{$prefix} . $suffix;
	}
	
	my $self = {
		'value' => $value ,
		'lang'  => (lc $lang) ,
		'dt'    => $dt ,
		'dturi' => $dturi ,
		};
	
	bless $self, $class;
}

1;
