Merge branch 'master' into autoconfig
[ikiwiki.git] / IkiWiki / Plugin / amazon_s3.pm
1 #!/usr/bin/perl
2 package IkiWiki::Plugin::amazon_s3;
3
4 use warnings;
5 no warnings 'redefine';
6 use strict;
7 use IkiWiki 2.00;
8 use IkiWiki::Render;
9 use Net::Amazon::S3;
10
11 # Store references to real subs before overriding them.
12 our %subs;
13 BEGIN {
14         foreach my $sub (qw{IkiWiki::writefile IkiWiki::prune}) {
15                 $subs{$sub}=\&$sub;
16         }
17 };
18
19 sub import { #{{{
20         hook(type => "getopt", id => "amazon_s3", call => \&getopt);
21         hook(type => "getsetup", id => "amazon_s3", call => \&getsetup);
22         hook(type => "checkconfig", id => "amazon_s3", call => \&checkconfig);
23 } # }}}
24
25 sub getopt () { #{{{
26         eval q{use Getopt::Long};
27         error($@) if $@;
28         Getopt::Long::Configure('pass_through');
29         GetOptions("delete-bucket" => sub {
30                 my $bucket=getbucket();
31                 debug(gettext("deleting bucket.."));
32                 my $resp = $bucket->list_all or die $bucket->err . ": " . $bucket->errstr;
33                 foreach my $key (@{$resp->{keys}}) {
34                         debug("\t".$key->{key});
35                         $bucket->delete_key($key->{key}) or die $bucket->err . ": " . $bucket->errstr;
36                 }
37                 $bucket->delete_bucket or die $bucket->err . ": " . $bucket->errstr;
38                 debug(gettext("done"));
39                 exit(0);
40         });
41 } #}}}
42
43 sub getsetup () { #{{{
44         return
45                  amazon_s3_key_id => {
46                         type => "string",
47                         default => "",
48                         description => "public access key id",
49                         safe => 1,
50                         rebuild => 0,
51                 },
52                 amazon_s3_key_id => {
53                         type => "string",
54                         default => "",
55                         description => "file holding secret key",
56                         safe => 0, # ikiwiki reads this file
57                         rebuild => 0,
58                 },
59                 amazon_s3_bucket => {
60                         type => "string",
61                         default => "",
62                         example => "mywiki",
63                         description => "globally unique name of bucket to store wiki in",
64                         safe => 1,
65                         rebuild => 1,
66                 },
67                 amazon_s3_prefix => {
68                         type => "string",
69                         default => "wiki/",
70                         description => "a prefix to prepend to each page name",
71                         safe => 1,
72                         rebuild => 1,
73                 },
74                 amazon_s3_location => {
75                         type => "string",
76                         default => "",
77                         example => "EU",
78                         description => "which S3 datacenter to use (leave blank for default)",
79                         safe => 1,
80                         rebuild => 1,
81                 },
82                 amazon_s3_dupindex => {
83                         type => "boolean",
84                         default => 0,
85                         description => "store each index file twice? (to allow urls ending in \"/index.html\" and \"/\")",
86                         safe => 1,
87                         rebuild => 1,
88                 },
89 } #}}}
90
91 sub checkconfig { #{{{
92         foreach my $field (qw{amazon_s3_key_id amazon_s3_key_file
93                               amazon_s3_bucket}) {
94                 if (! exists $config{$field} || ! defined $config{$field}) {
95                         error(sprintf(gettext("Must specify %s"), $field));
96                 }
97         }
98         if (! exists $config{amazon_s3_prefix} ||
99             ! defined $config{amazon_s3_prefix}) {
100             $config{amazon_s3_prefix}="wiki/";
101         }
102 } #}}}
103
104 {
105 my $bucket;
106 sub getbucket { #{{{
107         return $bucket if defined $bucket;
108         
109         open(IN, "<", $config{amazon_s3_key_file}) || error($config{amazon_s3_key_file}.": ".$!);
110         my $key=<IN>;
111         chomp $key;
112         close IN;
113
114         my $s3=Net::Amazon::S3->new({
115                 aws_access_key_id => $config{amazon_s3_key_id},
116                 aws_secret_access_key => $key,
117                 retry => 1,
118         });
119
120         # make sure the bucket exists
121         if (exists $config{amazon_s3_location}) {
122                 $bucket=$s3->add_bucket({
123                         bucket => $config{amazon_s3_bucket},
124                         location_constraint => $config{amazon_s3_location},
125                 });
126         }
127         else {
128                 $bucket=$s3->add_bucket({
129                         bucket => $config{amazon_s3_bucket},
130                 });
131         }
132
133         if (! $bucket) {
134                 error(gettext("Failed to create bucket in S3: ").
135                         $s3->err.": ".$s3->errstr."\n");
136         }
137
138         return $bucket;
139 } #}}}
140 }
141
142 # Given a file, return any S3 keys associated with it.
143 sub file2keys ($) { #{{{
144         my $file=shift;
145
146         my @keys;
147         if ($file =~ /^\Q$config{destdir}\/\E(.*)/) {
148                 push @keys, $config{amazon_s3_prefix}.$1;
149
150                 # Munge foo/index.html to foo/
151                 if ($keys[0]=~/(^|.*\/)index.$config{htmlext}$/) {
152                         # A duplicate might need to be stored under the
153                         # unmunged name too.
154                         if (!$config{usedirs} || $config{amazon_s3_dupindex}) {
155                                 push @keys, $1;
156                         }
157                         else {
158                                 @keys=($1);
159                         }
160                 }
161         }
162         return @keys;
163 } #}}}
164
165 package IkiWiki;
166 use File::MimeInfo;
167 use Encode;
168
169 # This is a wrapper around the real writefile.
170 sub writefile ($$$;$$) { #{{{
171         my $file=shift;
172         my $destdir=shift;
173         my $content=shift;
174         my $binary=shift;
175         my $writer=shift;
176
177         # First, write the file to disk.
178         my $ret=$IkiWiki::Plugin::amazon_s3::subs{'IkiWiki::writefile'}->($file, $destdir, $content, $binary, $writer);
179                 
180         my @keys=IkiWiki::Plugin::amazon_s3::file2keys("$destdir/$file");
181
182         # Store the data in S3.
183         if (@keys) {
184                 my $bucket=IkiWiki::Plugin::amazon_s3::getbucket();
185
186                 # The http layer tries to downgrade utf-8
187                 # content, but that can fail (see
188                 # http://rt.cpan.org/Ticket/Display.html?id=35710),
189                 # so force convert it to bytes.
190                 $content=encode_utf8($content) if defined $content;
191
192                 my %opts=(
193                         acl_short => 'public-read',
194                         content_type => mimetype("$destdir/$file"),
195                 );
196
197                 # If there are multiple keys to write, data is sent
198                 # multiple times.
199                 # TODO: investigate using the new copy operation.
200                 #       (It may not be robust enough.)
201                 foreach my $key (@keys) {
202                         my $res;
203                         if (! $writer) {
204                                 $res=$bucket->add_key($key, $content, \%opts);
205                         }
206                         else {
207                                 # This test for empty files is a workaround
208                                 # for this bug:
209                                 # http://rt.cpan.org//Ticket/Display.html?id=35731
210                                 if (-z "$destdir/$file") {
211                                         $res=$bucket->add_key($key, "", \%opts);
212                                 }
213                                 else {
214                                         # read back in the file that the writer emitted
215                                         $res=$bucket->add_key_filename($key, "$destdir/$file", \%opts);
216                                 }
217                         }
218                         if (! $res) {
219                                 error(gettext("Failed to save file to S3: ").
220                                         $bucket->err.": ".$bucket->errstr."\n");
221                         }
222                 }
223         }
224
225         return $ret;
226 } #}}}
227
228 # This is a wrapper around the real prune.
229 sub prune ($) { #{{{
230         my $file=shift;
231
232         my @keys=IkiWiki::Plugin::amazon_s3::file2keys($file);
233
234         # Prune files out of S3 too.
235         if (@keys) {
236                 my $bucket=IkiWiki::Plugin::amazon_s3::getbucket();
237
238                 foreach my $key (@keys) {
239                         my $res=$bucket->delete_key($key);
240                         if (! $res) {
241                                 error(gettext("Failed to delete file from S3: ").
242                                         $bucket->err.": ".$bucket->errstr."\n");
243                         }
244                 }
245         }
246
247         return $IkiWiki::Plugin::amazon_s3::subs{'IkiWiki::prune'}->($file);
248 } #}}}
249
250 1