IkiWiki/Plugin/amazon_s3.pm

   1 #!/usr/bin/perl
   2 package IkiWiki::Plugin::amazon_s3;
   3
   4 use warnings;
   5 no warnings 'redefine';
   6 use strict;
   7 use IkiWiki 2.00;
   8 use IkiWiki::Render;
   9 use Net::Amazon::S3;
  10
  11 # Store references to real subs before overriding them.
  12 our %subs;
  13 BEGIN {
  14         foreach my $sub (qw{IkiWiki::writefile IkiWiki::prune}) {
  15                 $subs{$sub}=\&$sub;
  16         }
  17 };
  18
  19 sub import { #{{{
  20         hook(type => "getopt", id => "amazon_s3", call => \&getopt);
  21         hook(type => "getsetup", id => "amazon_s3", call => \&getsetup);
  22         hook(type => "checkconfig", id => "amazon_s3", call => \&checkconfig);
  23 } # }}}
  24
  25 sub getopt () { #{{{
  26         eval q{use Getopt::Long};
  27         error($@) if $@;
  28         Getopt::Long::Configure('pass_through');
  29         GetOptions("delete-bucket" => sub {
  30                 my $bucket=getbucket();
  31                 debug(gettext("deleting bucket.."));
  32                 my $resp = $bucket->list_all or die $bucket->err . ": " . $bucket->errstr;
  33                 foreach my $key (@{$resp->{keys}}) {
  34                         debug("\t".$key->{key});
  35                         $bucket->delete_key($key->{key}) or die $bucket->err . ": " . $bucket->errstr;
  36                 }
  37                 $bucket->delete_bucket or die $bucket->err . ": " . $bucket->errstr;
  38                 debug(gettext("done"));
  39                 exit(0);
  40         });
  41 } #}}}
  42
  43 sub getsetup () { #{{{
  44         return
  45                  amazon_s3_key_id => {
  46                         type => "string",
  47                         default => "",
  48                         description => "public access key id",
  49                         safe => 1,
  50                         rebuild => 0,
  51                 },
  52                 amazon_s3_key_id => {
  53                         type => "string",
  54                         default => "",
  55                         description => "file holding secret key",
  56                         safe => 0, # ikiwiki reads this file
  57                         rebuild => 0,
  58                 },
  59                 amazon_s3_bucket => {
  60                         type => "string",
  61                         default => "",
  62                         example => "mywiki",
  63                         description => "globally unique name of bucket to store wiki in",
  64                         safe => 1,
  65                         rebuild => 1,
  66                 },
  67                 amazon_s3_prefix => {
  68                         type => "string",
  69                         default => "wiki/",
  70                         description => "a prefix to prepend to each page name",
  71                         safe => 1,
  72                         rebuild => 1,
  73                 },
  74                 amazon_s3_location => {
  75                         type => "string",
  76                         default => "",
  77                         example => "EU",
  78                         description => "which S3 datacenter to use (leave blank for default)",
  79                         safe => 1,
  80                         rebuild => 1,
  81                 },
  82                 amazon_s3_dupindex => {
  83                         type => "boolean",
  84                         default => 0,
  85                         description => "store each index file twice? (to allow urls ending in \"/index.html\" and \"/\")",
  86                         safe => 1,
  87                         rebuild => 1,
  88                 },
  89 } #}}}
  90
  91 sub checkconfig { #{{{
  92         foreach my $field (qw{amazon_s3_key_id amazon_s3_key_file
  93                               amazon_s3_bucket}) {
  94                 if (! exists $config{$field} || ! defined $config{$field}) {
  95                         error(sprintf(gettext("Must specify %s"), $field));
  96                 }
  97         }
  98         if (! exists $config{amazon_s3_prefix} ||
  99             ! defined $config{amazon_s3_prefix}) {
 100             $config{amazon_s3_prefix}="wiki/";
 101         }
 102 } #}}}
 103
 104 {
 105 my $bucket;
 106 sub getbucket { #{{{
 107         return $bucket if defined $bucket;
 108
 109         open(IN, "<", $config{amazon_s3_key_file}) || error($config{amazon_s3_key_file}.": ".$!);
 110         my $key=<IN>;
 111         chomp $key;
 112         close IN;
 113
 114         my $s3=Net::Amazon::S3->new({
 115                 aws_access_key_id => $config{amazon_s3_key_id},
 116                 aws_secret_access_key => $key,
 117                 retry => 1,
 118         });
 119
 120         # make sure the bucket exists
 121         if (exists $config{amazon_s3_location}) {
 122                 $bucket=$s3->add_bucket({
 123                         bucket => $config{amazon_s3_bucket},
 124                         location_constraint => $config{amazon_s3_location},
 125                 });
 126         }
 127         else {
 128                 $bucket=$s3->add_bucket({
 129                         bucket => $config{amazon_s3_bucket},
 130                 });
 131         }
 132
 133         if (! $bucket) {
 134                 error(gettext("Failed to create bucket in S3: ").
 135                         $s3->err.": ".$s3->errstr."\n");
 136         }
 137
 138         return $bucket;
 139 } #}}}
 140 }
 141
 142 # Given a file, return any S3 keys associated with it.
 143 sub file2keys ($) { #{{{
 144         my $file=shift;
 145
 146         my @keys;
 147         if ($file =~ /^\Q$config{destdir}\/\E(.*)/) {
 148                 push @keys, $config{amazon_s3_prefix}.$1;
 149
 150                 # Munge foo/index.html to foo/
 151                 if ($keys[0]=~/(^|.*\/)index.$config{htmlext}$/) {
 152                         # A duplicate might need to be stored under the
 153                         # unmunged name too.
 154                         if (!$config{usedirs} || $config{amazon_s3_dupindex}) {
 155                                 push @keys, $1;
 156                         }
 157                         else {
 158                                 @keys=($1);
 159                         }
 160                 }
 161         }
 162         return @keys;
 163 } #}}}
 164
 165 package IkiWiki;
 166 use File::MimeInfo;
 167 use Encode;
 168
 169 # This is a wrapper around the real writefile.
 170 sub writefile ($$$;$$) { #{{{
 171         my $file=shift;
 172         my $destdir=shift;
 173         my $content=shift;
 174         my $binary=shift;
 175         my $writer=shift;
 176
 177         # First, write the file to disk.
 178         my $ret=$IkiWiki::Plugin::amazon_s3::subs{'IkiWiki::writefile'}->($file, $destdir, $content, $binary, $writer);
 179
 180         my @keys=IkiWiki::Plugin::amazon_s3::file2keys("$destdir/$file");
 181
 182         # Store the data in S3.
 183         if (@keys) {
 184                 my $bucket=IkiWiki::Plugin::amazon_s3::getbucket();
 185
 186                 # The http layer tries to downgrade utf-8
 187                 # content, but that can fail (see
 188                 # http://rt.cpan.org/Ticket/Display.html?id=35710),
 189                 # so force convert it to bytes.
 190                 $content=encode_utf8($content) if defined $content;
 191
 192                 my %opts=(
 193                         acl_short => 'public-read',
 194                         content_type => mimetype("$destdir/$file"),
 195                 );
 196
 197                 # If there are multiple keys to write, data is sent
 198                 # multiple times.
 199                 # TODO: investigate using the new copy operation.
 200                 #       (It may not be robust enough.)
 201                 foreach my $key (@keys) {
 202                         my $res;
 203                         if (! $writer) {
 204                                 $res=$bucket->add_key($key, $content, \%opts);
 205                         }
 206                         else {
 207                                 # This test for empty files is a workaround
 208                                 # for this bug:
 209                                 # http://rt.cpan.org//Ticket/Display.html?id=35731
 210                                 if (-z "$destdir/$file") {
 211                                         $res=$bucket->add_key($key, "", \%opts);
 212                                 }
 213                                 else {
 214                                         # read back in the file that the writer emitted
 215                                         $res=$bucket->add_key_filename($key, "$destdir/$file", \%opts);
 216                                 }
 217                         }
 218                         if (! $res) {
 219                                 error(gettext("Failed to save file to S3: ").
 220                                         $bucket->err.": ".$bucket->errstr."\n");
 221                         }
 222                 }
 223         }
 224
 225         return $ret;
 226 } #}}}
 227
 228 # This is a wrapper around the real prune.
 229 sub prune ($) { #{{{
 230         my $file=shift;
 231
 232         my @keys=IkiWiki::Plugin::amazon_s3::file2keys($file);
 233
 234         # Prune files out of S3 too.
 235         if (@keys) {
 236                 my $bucket=IkiWiki::Plugin::amazon_s3::getbucket();
 237
 238                 foreach my $key (@keys) {
 239                         my $res=$bucket->delete_key($key);
 240                         if (! $res) {
 241                                 error(gettext("Failed to delete file from S3: ").
 242                                         $bucket->err.": ".$bucket->errstr."\n");
 243                         }
 244                 }
 245         }
 246
 247         return $IkiWiki::Plugin::amazon_s3::subs{'IkiWiki::prune'}->($file);
 248 } #}}}
 249
 250 1