2 # .po as a wiki page type
3 # Licensed under GPL v2 or greater
4 # Copyright (C) 2008 intrigeri <intrigeri@boum.org>
5 # inspired by the GPL'd po4a-translate,
6 # which is Copyright 2002, 2003, 2004 by Martin Quinson (mquinson#debian.org)
7 package IkiWiki::Plugin::po;
13 use Locale::Po4a::Chooser;
26 memoize("istranslatable");
27 memoize("_istranslation");
28 memoize("percenttranslated");
31 hook(type => "getsetup", id => "po", call => \&getsetup);
32 hook(type => "checkconfig", id => "po", call => \&checkconfig);
33 hook(type => "needsbuild", id => "po", call => \&needsbuild);
34 hook(type => "scan", id => "po", call => \&scan, last =>1);
35 hook(type => "filter", id => "po", call => \&filter);
36 hook(type => "htmlize", id => "po", call => \&htmlize);
37 hook(type => "pagetemplate", id => "po", call => \&pagetemplate, last => 1);
38 hook(type => "change", id => "po", call => \&change);
39 hook(type => "editcontent", id => "po", call => \&editcontent);
41 $origsubs{'bestlink'}=\&IkiWiki::bestlink;
42 inject(name => "IkiWiki::bestlink", call => \&mybestlink);
43 $origsubs{'beautify_urlpath'}=\&IkiWiki::beautify_urlpath;
44 inject(name => "IkiWiki::beautify_urlpath", call => \&mybeautify_urlpath);
45 $origsubs{'targetpage'}=\&IkiWiki::targetpage;
46 inject(name => "IkiWiki::targetpage", call => \&mytargetpage);
47 $origsubs{'urlto'}=\&IkiWiki::urlto;
48 inject(name => "IkiWiki::urlto", call => \&myurlto);
57 # 2. Injected functions
58 # 3. Blackboxes for private data
67 sub getsetup () { #{{{
73 po_master_language => {
79 description => "master language (non-PO files)",
83 po_slave_languages => {
90 description => "slave languages (PO files)",
94 po_translatable_pages => {
96 example => "!*/Discussion",
97 description => "PageSpec controlling which pages are translatable",
98 link => "ikiwiki/PageSpec",
104 example => "current",
105 description => "internal linking behavior (default/current/negotiated)",
111 sub checkconfig () { #{{{
112 foreach my $field (qw{po_master_language po_slave_languages}) {
113 if (! exists $config{$field} || ! defined $config{$field}) {
114 error(sprintf(gettext("Must specify %s"), $field));
117 if (! (keys %{$config{po_slave_languages}})) {
118 error(gettext("At least one slave language must be defined in po_slave_languages"));
122 or error(sprintf(gettext("%s is not a valid language code"), $_));
123 } ($config{po_master_language}{code}, keys %{$config{po_slave_languages}});
124 if (! exists $config{po_translatable_pages} ||
125 ! defined $config{po_translatable_pages}) {
126 $config{po_translatable_pages}="";
128 if (! exists $config{po_link_to} ||
129 ! defined $config{po_link_to}) {
130 $config{po_link_to}='default';
133 $config{po_link_to} eq $_
134 } ('default', 'current', 'negotiated')) {
135 warn(sprintf(gettext('po_link_to=%s is not a valid setting, falling back to po_link_to=default'),
136 $config{po_link_to}));
137 $config{po_link_to}='default';
139 elsif ($config{po_link_to} eq "negotiated" && ! $config{usedirs}) {
140 warn(gettext('po_link_to=negotiated requires usedirs to be enabled, falling back to po_link_to=default'));
141 $config{po_link_to}='default';
143 push @{$config{wiki_file_prune_regexps}}, qr/\.pot$/;
146 sub needsbuild () { #{{{
147 my $needsbuild=shift;
149 # backup @needsbuild content so that change() can know whether
150 # a given master page was rendered because its source file was changed
151 @origneedsbuild=(@$needsbuild);
153 buildtranslationscache();
155 # make existing translations depend on the corresponding master page
156 foreach my $master (keys %translations) {
157 map add_depends($_, $master), values %{otherlanguages($master)};
161 # Massage the recorded state of internal links so that:
162 # - it matches the actually generated links, rather than the links as written
163 # in the pages' source
164 # - backlinks are consistent in all cases
167 my $page=$params{page};
168 my $content=$params{content};
170 return unless UNIVERSAL::can("IkiWiki::Plugin::link", "import");
172 if (istranslation($page)) {
173 foreach my $destpage (@{$links{$page}}) {
174 if (istranslatable($destpage)) {
175 # replace one occurence of $destpage in $links{$page}
176 # (we only want to replace the one that was added by
177 # IkiWiki::Plugin::link::scan, other occurences may be
178 # there for other reasons)
179 for (my $i=0; $i<@{$links{$page}}; $i++) {
180 if (@{$links{$page}}[$i] eq $destpage) {
181 @{$links{$page}}[$i] = $destpage . '.' . lang($page);
188 elsif (! istranslatable($page) && ! istranslation($page)) {
189 foreach my $destpage (@{$links{$page}}) {
190 if (istranslatable($destpage)) {
191 # make sure any destpage's translations has
192 # $page in its backlinks
193 push @{$links{$page}},
194 values %{otherlanguages($destpage)};
200 # We use filter to convert PO to the master page's format,
201 # since the rest of ikiwiki should not work on PO files.
202 sub filter (@) { #{{{
205 my $page = $params{page};
206 my $destpage = $params{destpage};
207 my $content = decode_utf8(encode_utf8($params{content}));
209 return $content if ( ! istranslation($page)
210 || alreadyfiltered($page, $destpage) );
212 # CRLF line terminators make poor Locale::Po4a feel bad
213 $content=~s/\r\n/\n/g;
215 # Implementation notes
217 # 1. Locale::Po4a reads/writes from/to files, and I'm too lazy
218 # to learn how to disguise a variable as a file.
219 # 2. There are incompatibilities between some File::Temp versions
220 # (including 0.18, bundled with Lenny's perl-modules package)
221 # and others (e.g. 0.20, previously present in the archive as
222 # a standalone package): under certain circumstances, some
223 # return a relative filename, whereas others return an absolute one;
224 # we here use this module in a way that is at least compatible
225 # with 0.18 and 0.20. Beware, hit'n'run refactorers!
226 my $infile = new File::Temp(TEMPLATE => "ikiwiki-po-filter-in.XXXXXXXXXX",
227 DIR => File::Spec->tmpdir,
228 UNLINK => 1)->filename;
229 my $outfile = new File::Temp(TEMPLATE => "ikiwiki-po-filter-out.XXXXXXXXXX",
230 DIR => File::Spec->tmpdir,
231 UNLINK => 1)->filename;
233 writefile(basename($infile), File::Spec->tmpdir, $content);
235 my $masterfile = srcfile($pagesources{masterpage($page)});
238 push @masters,$masterfile;
240 "markdown" => (pagetype($masterfile) eq 'mdwn') ? 1 : 0,
242 my $doc=Locale::Po4a::Chooser::new('text',%options);
244 'po_in_name' => \@pos,
245 'file_in_name' => \@masters,
246 'file_in_charset' => 'utf-8',
247 'file_out_charset' => 'utf-8',
248 ) or error("[po/filter:$page]: failed to translate");
249 $doc->write($outfile) or error("[po/filter:$page] could not write $outfile");
250 $content = readfile($outfile) or error("[po/filter:$page] could not read $outfile");
252 # Unlinking should happen automatically, thanks to File::Temp,
253 # but it does not work here, probably because of the way writefile()
254 # and Locale::Po4a::write() work.
255 unlink $infile, $outfile;
257 setalreadyfiltered($page, $destpage);
261 sub htmlize (@) { #{{{
264 my $page = $params{page};
265 my $content = $params{content};
267 # ignore PO files this plugin did not create
268 return $content unless istranslation($page);
270 # force content to be htmlize'd as if it was the same type as the master page
271 return IkiWiki::htmlize($page, $page,
272 pagetype(srcfile($pagesources{masterpage($page)})),
276 sub pagetemplate (@) { #{{{
278 my $page=$params{page};
279 my $destpage=$params{destpage};
280 my $template=$params{template};
282 my ($masterpage, $lang) = istranslation($page);
284 if (istranslation($page) && $template->query(name => "percenttranslated")) {
285 $template->param(percenttranslated => percenttranslated($page));
287 if ($template->query(name => "istranslation")) {
288 $template->param(istranslation => scalar istranslation($page));
290 if ($template->query(name => "istranslatable")) {
291 $template->param(istranslatable => istranslatable($page));
293 if ($template->query(name => "HOMEPAGEURL")) {
294 $template->param(homepageurl => homepageurl($page));
296 if ($template->query(name => "otherlanguages")) {
297 $template->param(otherlanguages => [otherlanguagesloop($page)]);
298 map add_depends($page, $_), (values %{otherlanguages($page)});
300 # Rely on IkiWiki::Render's genpage() to decide wether
301 # a discussion link should appear on $page; this is not
302 # totally accurate, though: some broken links may be generated
303 # when cgiurl is disabled.
304 # This compromise avoids some code duplication, and will probably
305 # prevent future breakage when ikiwiki internals change.
306 # Known limitations are preferred to future random bugs.
307 if ($template->param('discussionlink') && istranslation($page)) {
308 $template->param('discussionlink' => htmllink(
311 $masterpage . '/' . gettext("Discussion"),
314 linktext => gettext("Discussion"),
317 # Remove broken parentlink to ./index.html on home page's translations.
318 # It works because this hook has the "last" parameter set, to ensure it
319 # runs after parentlinks' own pagetemplate hook.
320 if ($template->param('parentlinks')
321 && istranslation($page)
322 && $masterpage eq "index") {
323 $template->param('parentlinks' => []);
330 my $updated_po_files=0;
332 # Refresh/create POT and PO files as needed.
333 foreach my $page (map pagename($_), @rendered) {
334 next unless istranslatable($page);
335 my $file=srcfile($pagesources{$page});
336 my $updated_pot_file=0;
337 # Only refresh Pot file if it does not exist, or if
338 # $pagesources{$page} was changed: don't if only the HTML was
339 # refreshed, e.g. because of a dependency.
340 if ((grep { $_ eq $pagesources{$page} } @origneedsbuild)
341 || ! -e potfile($file)) {
347 push @pofiles, $_ if ($updated_pot_file || ! -e $_);
350 refreshpofiles($file, @pofiles);
351 map { IkiWiki::rcs_add($_); } @pofiles if ($config{rcs});
356 if ($updated_po_files) {
357 # Check staged changes in.
359 IkiWiki::disable_commit_hook();
360 IkiWiki::rcs_commit_staged(gettext("updated PO files"),
361 "IkiWiki::Plugin::po::change", "127.0.0.1");
362 IkiWiki::enable_commit_hook();
363 IkiWiki::rcs_update();
365 # Reinitialize module's private variables.
366 resetalreadyfiltered();
367 resettranslationscache();
369 # Trigger a wiki refresh.
370 require IkiWiki::Render;
371 # without preliminary saveindex/loadindex, refresh()
372 # complains about a lot of uninitialized variables
373 IkiWiki::saveindex();
374 IkiWiki::loadindex();
376 IkiWiki::saveindex();
380 # As we're previewing or saving a page, the content may have
381 # changed, so tell the next filter() invocation it must not be lazy.
382 sub editcontent () { #{{{
385 unsetalreadyfiltered($params{page}, $params{page});
386 return $params{content};
391 # | Injected functions
394 # Implement po_link_to 'current' and 'negotiated' settings.
395 sub mybestlink ($$) { #{{{
399 my $res=$origsubs{'bestlink'}->(masterpage($page), $link);
401 && ($config{po_link_to} eq "current" || $config{po_link_to} eq "negotiated")
402 && istranslatable($res)
403 && istranslation($page)) {
404 return $res . "." . lang($page);
409 sub mybeautify_urlpath ($) { #{{{
412 my $res=$origsubs{'beautify_urlpath'}->($url);
413 if ($config{po_link_to} eq "negotiated") {
414 $res =~ s!/\Qindex.$config{po_master_language}{code}.$config{htmlext}\E$!/!;
415 $res =~ s!/\Qindex.$config{htmlext}\E$!/!;
417 $res =~ s!/\Qindex.$_.$config{htmlext}\E$!/!;
418 } (keys %{$config{po_slave_languages}});
423 sub mytargetpage ($$) { #{{{
427 if (istranslation($page) || istranslatable($page)) {
428 my ($masterpage, $lang) = (masterpage($page), lang($page));
429 if (! $config{usedirs} || $masterpage eq 'index') {
430 return $masterpage . "." . $lang . "." . $ext;
433 return $masterpage . "/index." . $lang . "." . $ext;
436 return $origsubs{'targetpage'}->($page, $ext);
439 sub myurlto ($$;$) { #{{{
444 # workaround hard-coded /index.$config{htmlext} in IkiWiki::urlto()
446 && $config{po_link_to} eq "current"
447 && istranslatable('index')) {
448 return IkiWiki::beautify_urlpath(IkiWiki::baseurl($from) . "index." . lang($from) . ".$config{htmlext}");
450 return $origsubs{'urlto'}->($to,$from,$absolute);
455 # | Blackboxes for private data
461 sub alreadyfiltered($$) { #{{{
465 return ( exists $filtered{$page}{$destpage}
466 && $filtered{$page}{$destpage} eq 1 );
469 sub setalreadyfiltered($$) { #{{{
473 $filtered{$page}{$destpage}=1;
476 sub unsetalreadyfiltered($$) { #{{{
480 if (exists $filtered{$page}{$destpage}) {
481 delete $filtered{$page}{$destpage};
485 sub resetalreadyfiltered() { #{{{
495 sub maybe_add_leading_slash ($;$) { #{{{
498 $add=1 unless defined $add;
499 return '/' . $str if $add;
503 sub istranslatable ($) { #{{{
507 my $file=$pagesources{$page};
509 return 0 unless defined $file;
510 return 0 if (defined pagetype($file) && pagetype($file) eq 'po');
511 return 0 if $file =~ /\.pot$/;
512 return 1 if pagespec_match($page, $config{po_translatable_pages});
516 sub _istranslation ($) { #{{{
519 my $hasleadingslash = ($page=~s#^/##);
520 my $file=$pagesources{$page};
521 return 0 unless (defined $file
522 && defined pagetype($file)
523 && pagetype($file) eq 'po');
524 return 0 if $file =~ /\.pot$/;
526 my ($masterpage, $lang) = ($page =~ /(.*)[.]([a-z]{2})$/);
527 return 0 unless (defined $masterpage && defined $lang
528 && length $masterpage && length $lang
529 && defined $pagesources{$masterpage}
530 && defined $config{po_slave_languages}{$lang});
532 return (maybe_add_leading_slash($masterpage, $hasleadingslash), $lang)
533 if istranslatable($masterpage);
536 sub istranslation ($) { #{{{
539 if (1 < (my ($masterpage, $lang) = _istranslation($page))) {
540 my $hasleadingslash = ($masterpage=~s#^/##);
541 $translations{$masterpage}{$lang}=$page unless exists $translations{$masterpage}{$lang};
542 return (maybe_add_leading_slash($masterpage, $hasleadingslash), $lang);
547 sub masterpage ($) { #{{{
550 if ( 1 < (my ($masterpage, $lang) = _istranslation($page))) {
559 if (1 < (my ($masterpage, $lang) = _istranslation($page))) {
562 return $config{po_master_language}{code};
565 sub islanguagecode ($) { #{{{
568 return ($code =~ /^[a-z]{2}$/);
571 sub otherlanguages($) { #{{{
575 if (istranslatable($page)) {
576 %ret = %{$translations{$page}} if defined $translations{$page};
578 elsif (istranslation($page)) {
579 my $masterpage = masterpage($page);
580 $ret{$config{po_master_language}{code}} = $masterpage;
581 foreach my $lang (sort keys %{$translations{$masterpage}}) {
582 next if $lang eq lang($page);
583 $ret{$lang} = $translations{$masterpage}{$lang};
589 sub potfile ($) { #{{{
590 my $masterfile=shift;
592 (my $name, my $dir, my $suffix) = fileparse($masterfile, qr/\.[^.]*/);
593 $dir='' if $dir eq './';
594 return File::Spec->catpath('', $dir, $name . ".pot");
597 sub pofile ($$) { #{{{
598 my $masterfile=shift;
601 (my $name, my $dir, my $suffix) = fileparse($masterfile, qr/\.[^.]*/);
602 $dir='' if $dir eq './';
603 return File::Spec->catpath('', $dir, $name . "." . $lang . ".po");
606 sub pofiles ($) { #{{{
607 my $masterfile=shift;
609 return map pofile($masterfile, $_), (keys %{$config{po_slave_languages}});
612 sub refreshpot ($) { #{{{
613 my $masterfile=shift;
615 my $potfile=potfile($masterfile);
616 my %options = ("markdown" => (pagetype($masterfile) eq 'mdwn') ? 1 : 0);
617 my $doc=Locale::Po4a::Chooser::new('text',%options);
618 $doc->{TT}{utf_mode} = 1;
619 $doc->{TT}{file_in_charset} = 'utf-8';
620 $doc->{TT}{file_out_charset} = 'utf-8';
621 $doc->read($masterfile);
622 # let's cheat a bit to force porefs option to be passed to Locale::Po4a::Po;
623 # this is undocument use of internal Locale::Po4a::TransTractor's data,
624 # compulsory since this module prevents us from using the porefs option.
625 my %po_options = ('porefs' => 'none');
626 $doc->{TT}{po_out}=Locale::Po4a::Po->new(\%po_options);
627 $doc->{TT}{po_out}->set_charset('utf-8');
630 IkiWiki::prep_writefile(basename($potfile),dirname($potfile));
631 $doc->writepo($potfile);
634 sub refreshpofiles ($@) { #{{{
635 my $masterfile=shift;
638 my $potfile=potfile($masterfile);
639 error("[po/refreshpofiles] POT file ($potfile) does not exist") unless (-e $potfile);
641 foreach my $pofile (@pofiles) {
642 IkiWiki::prep_writefile(basename($pofile),dirname($pofile));
644 system("msgmerge", "-U", "--backup=none", $pofile, $potfile) == 0
645 or error("[po/refreshpofiles:$pofile] failed to update");
648 File::Copy::syscopy($potfile,$pofile)
649 or error("[po/refreshpofiles:$pofile] failed to copy the POT file");
654 sub buildtranslationscache() { #{{{
655 # use istranslation's side-effect
656 map istranslation($_), (keys %pagesources);
659 sub resettranslationscache() { #{{{
663 sub flushmemoizecache() { #{{{
664 Memoize::flush_cache("istranslatable");
665 Memoize::flush_cache("_istranslation");
666 Memoize::flush_cache("percenttranslated");
669 sub urlto_with_orig_beautiful_urlpath($$) { #{{{
673 inject(name => "IkiWiki::beautify_urlpath", call => $origsubs{'beautify_urlpath'});
674 my $res=urlto($to, $from);
675 inject(name => "IkiWiki::beautify_urlpath", call => \&mybeautify_urlpath);
680 sub percenttranslated ($) { #{{{
683 return gettext("N/A") unless istranslation($page);
684 my $file=srcfile($pagesources{$page});
685 my $masterfile = srcfile($pagesources{masterpage($page)});
688 push @masters,$masterfile;
690 "markdown" => (pagetype($masterfile) eq 'mdwn') ? 1 : 0,
692 my $doc=Locale::Po4a::Chooser::new('text',%options);
694 'po_in_name' => \@pos,
695 'file_in_name' => \@masters,
696 'file_in_charset' => 'utf-8',
697 'file_out_charset' => 'utf-8',
698 ) or error("[po/percenttranslated:$page]: failed to translate");
699 my ($percent,$hit,$queries) = $doc->stats();
703 sub languagename ($) { #{{{
706 return $config{po_master_language}{name}
707 if $code eq $config{po_master_language}{code};
708 return $config{po_slave_languages}{$code}
709 if defined $config{po_slave_languages}{$code};
713 sub otherlanguagesloop ($) { #{{{
717 my %otherpages=%{otherlanguages($page)};
718 while (my ($lang, $otherpage) = each %otherpages) {
719 if (istranslation($page) && masterpage($page) eq $otherpage) {
721 url => urlto_with_orig_beautiful_urlpath($otherpage, $page),
723 language => languagename($lang),
729 url => urlto_with_orig_beautiful_urlpath($otherpage, $page),
731 language => languagename($lang),
732 percent => percenttranslated($otherpage),
737 return -1 if $a->{code} eq $config{po_master_language}{code};
738 return 1 if $b->{code} eq $config{po_master_language}{code};
739 return $a->{language} cmp $b->{language};
743 sub homepageurl (;$) { #{{{
746 return urlto('', $page);
753 package IkiWiki::PageSpec;
758 sub match_istranslation ($;@) { #{{{
761 if (IkiWiki::Plugin::po::istranslation($page)) {
762 return IkiWiki::SuccessReason->new("is a translation page");
765 return IkiWiki::FailReason->new("is not a translation page");
769 sub match_istranslatable ($;@) { #{{{
772 if (IkiWiki::Plugin::po::istranslatable($page)) {
773 return IkiWiki::SuccessReason->new("is set as translatable in po_translatable_pages");
776 return IkiWiki::FailReason->new("is not set as translatable in po_translatable_pages");
780 sub match_lang ($$;@) { #{{{
784 my $regexp=IkiWiki::glob2re($wanted);
785 my $lang=IkiWiki::Plugin::po::lang($page);
786 if ($lang!~/^$regexp$/i) {
787 return IkiWiki::FailReason->new("file language is $lang, not $wanted");
790 return IkiWiki::SuccessReason->new("file language is $wanted");
794 sub match_currentlang ($$;@) { #{{{
799 return IkiWiki::FailReason->new("no location provided") unless exists $params{location};
801 my $currentlang=IkiWiki::Plugin::po::lang($params{location});
802 my $lang=IkiWiki::Plugin::po::lang($page);
804 if ($lang eq $currentlang) {
805 return IkiWiki::SuccessReason->new("file language is the same as current one, i.e. $currentlang");
808 return IkiWiki::FailReason->new("file language is $lang, whereas current language is $currentlang");