GDAL
cpl_vsil_curl_class.h
1/******************************************************************************
2 *
3 * Project: CPL - Common Portability Library
4 * Purpose: Declarations for /vsicurl/ and related file systems
5 * Author: Even Rouault, even.rouault at spatialys.com
6 *
7 ******************************************************************************
8 * Copyright (c) 2010-2018, Even Rouault <even.rouault at spatialys.com>
9 *
10 * Permission is hereby granted, free of charge, to any person obtaining a
11 * copy of this software and associated documentation files (the "Software"),
12 * to deal in the Software without restriction, including without limitation
13 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
14 * and/or sell copies of the Software, and to permit persons to whom the
15 * Software is furnished to do so, subject to the following conditions:
16 *
17 * The above copyright notice and this permission notice shall be included
18 * in all copies or substantial portions of the Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
25 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
26 * DEALINGS IN THE SOFTWARE.
27 ****************************************************************************/
28
29#ifndef CPL_VSIL_CURL_CLASS_H_INCLUDED
30#define CPL_VSIL_CURL_CLASS_H_INCLUDED
31
32#ifdef HAVE_CURL
33
34#include "cpl_aws.h"
35#include "cpl_port.h"
36#include "cpl_string.h"
37#include "cpl_vsil_curl_priv.h"
38#include "cpl_mem_cache.h"
39
40#include <curl/curl.h>
41
42#include <set>
43#include <map>
44#include <memory>
45
47
48// 7.18.1
49#if LIBCURL_VERSION_NUM >= 0x071201
50#define HAVE_CURLINFO_REDIRECT_URL
51#endif
52
53void VSICurlStreamingClearCache( void ); // from cpl_vsil_curl_streaming.cpp
54
55struct curl_slist* VSICurlSetOptions(CURL* hCurlHandle, const char* pszURL,
56 const char * const* papszOptions);
57struct curl_slist* VSICurlMergeHeaders( struct curl_slist* poDest,
58 struct curl_slist* poSrcToDestroy );
59
60namespace cpl {
61
62typedef enum
63{
64 EXIST_UNKNOWN = -1,
65 EXIST_NO,
66 EXIST_YES,
67} ExistStatus;
68
69class FileProp
70{
71 public:
72 ExistStatus eExists = EXIST_UNKNOWN;
73 vsi_l_offset fileSize = 0;
74 time_t mTime = 0;
75 time_t nExpireTimestampLocal = 0;
76 CPLString osRedirectURL{};
77 bool bHasComputedFileSize = false;
78 bool bIsDirectory = false;
79 bool bS3LikeRedirect = false;
80 CPLString ETag{};
81};
82
83typedef struct
84{
85 bool bGotFileList = false;
86 CPLStringList oFileList{}; /* only file name without path */
87} CachedDirList;
88
89typedef struct
90{
91 char* pBuffer = nullptr;
92 size_t nSize = 0;
93 bool bIsHTTP = false;
94 bool bIsInHeader = false;
95 bool bMultiRange = false;
96 vsi_l_offset nStartOffset = 0;
97 vsi_l_offset nEndOffset = 0;
98 int nHTTPCode = 0;
99 vsi_l_offset nContentLength = 0;
100 bool bFoundContentRange = false;
101 bool bError = false;
102 bool bDownloadHeaderOnly = false;
103 bool bDetectRangeDownloadingError = false;
104 GIntBig nTimestampDate = 0; // Corresponds to Date: header field
105
106 VSILFILE *fp = nullptr;
107 VSICurlReadCbkFunc pfnReadCbk = nullptr;
108 void *pReadCbkUserData = nullptr;
109 bool bInterrupted = false;
110
111#if LIBCURL_VERSION_NUM < 0x073600
112 // Workaround to ignore extra HTTP response headers from
113 // proxies in older versions of curl.
114 // CURLOPT_SUPPRESS_CONNECT_HEADERS fixes this
115 bool bIsProxyConnectHeader = false;
116#endif
117} WriteFuncStruct;
118
119/************************************************************************/
120/* VSICurlFilesystemHandler */
121/************************************************************************/
122
123class VSICurlHandle;
124
125class VSICurlFilesystemHandler : public VSIFilesystemHandler
126{
127 CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandler)
128
129 struct FilenameOffsetPair
130 {
131 std::string filename_;
132 vsi_l_offset offset_;
133
134 FilenameOffsetPair(const std::string& filename,
135 vsi_l_offset offset) :
136 filename_(filename), offset_(offset) {}
137
138 bool operator==(const FilenameOffsetPair& other) const
139 {
140 return filename_ == other.filename_ &&
141 offset_ == other.offset_;
142 }
143 };
144 struct FilenameOffsetPairHasher
145 {
146 std::size_t operator()(const FilenameOffsetPair& k) const
147 {
148 return std::hash<std::string>()(k.filename_) ^
149 std::hash<vsi_l_offset>()(k.offset_);
150 }
151 };
152
153 using RegionCacheType =
154 lru11::Cache<FilenameOffsetPair, std::shared_ptr<std::string>,
155 lru11::NullLock,
156 std::unordered_map<
157 FilenameOffsetPair,
158 typename std::list<lru11::KeyValuePair<FilenameOffsetPair,
159 std::shared_ptr<std::string>>>::iterator,
160 FilenameOffsetPairHasher>>;
161
162 RegionCacheType oRegionCache;
163
164 lru11::Cache<std::string, FileProp> oCacheFileProp;
165
166 int nCachedFilesInDirList = 0;
167 lru11::Cache<std::string, CachedDirList> oCacheDirList;
168
169 char** ParseHTMLFileList(const char* pszFilename,
170 int nMaxFiles,
171 char* pszData,
172 bool* pbGotFileList);
173
174protected:
175 CPLMutex *hMutex = nullptr;
176
177 virtual VSICurlHandle* CreateFileHandle(const char* pszFilename);
178 virtual char** GetFileList(const char *pszFilename,
179 int nMaxFiles,
180 bool* pbGotFileList);
181
182 void RegisterEmptyDir( const CPLString& osDirname );
183
184 bool AnalyseS3FileList( const CPLString& osBaseURL,
185 const char* pszXML,
186 CPLStringList& osFileList,
187 int nMaxFiles,
188 bool bIgnoreGlacierStorageClass,
189 bool& bIsTruncated );
190
191 void AnalyseSwiftFileList( const CPLString& osBaseURL,
192 const CPLString& osPrefix,
193 const char* pszJson,
194 CPLStringList& osFileList,
195 int nMaxFilesThisQuery,
196 int nMaxFiles,
197 bool& bIsTruncated,
198 CPLString& osNextMarker );
199
200 static const char* GetOptionsStatic();
201
202 static bool IsAllowedFilename( const char* pszFilename );
203
204public:
205 VSICurlFilesystemHandler();
206 ~VSICurlFilesystemHandler() override;
207
208 VSIVirtualHandle *Open( const char *pszFilename,
209 const char *pszAccess,
210 bool bSetError ) override;
211
212 int Stat( const char *pszFilename, VSIStatBufL *pStatBuf,
213 int nFlags ) override;
214 int Unlink( const char *pszFilename ) override;
215 int Rename( const char *oldpath, const char *newpath ) override;
216 int Mkdir( const char *pszDirname, long nMode ) override;
217 int Rmdir( const char *pszDirname ) override;
218 char **ReadDir( const char *pszDirname ) override
219 { return ReadDirEx(pszDirname, 0); }
220 char **ReadDirEx( const char *pszDirname, int nMaxFiles ) override;
221
222 int HasOptimizedReadMultiRange( const char* /* pszPath */ )
223 override { return true; }
224
225 const char* GetActualURL(const char* pszFilename) override;
226
227 const char* GetOptions() override;
228
229 char **ReadDirInternal( const char *pszDirname, int nMaxFiles,
230 bool* pbGotFileList );
231 void InvalidateDirContent( const char *pszDirname );
232
233 virtual CPLString GetFSPrefix() { return "/vsicurl/"; }
234 virtual bool AllowCachedDataFor(const char* pszFilename);
235
236 std::shared_ptr<std::string> GetRegion( const char* pszURL,
237 vsi_l_offset nFileOffsetStart );
238
239 void AddRegion( const char* pszURL,
240 vsi_l_offset nFileOffsetStart,
241 size_t nSize,
242 const char *pData );
243
244 bool GetCachedFileProp( const char* pszURL,
245 FileProp& oFileProp );
246 void SetCachedFileProp( const char* pszURL,
247 const FileProp& oFileProp );
248 void InvalidateCachedData( const char* pszURL );
249
250 CURLM *GetCurlMultiHandleFor( const CPLString& osURL );
251
252 virtual void ClearCache();
253 virtual void PartialClearCache(const char* pszFilename);
254
255
256 bool GetCachedDirList( const char* pszURL,
257 CachedDirList& oCachedDirList );
258 void SetCachedDirList( const char* pszURL,
259 const CachedDirList& oCachedDirList );
260 bool ExistsInCacheDirList( const CPLString& osDirname, bool *pbIsDir );
261
262 virtual CPLString GetURLFromFilename( const CPLString& osFilename );
263};
264
265/************************************************************************/
266/* VSICurlHandle */
267/************************************************************************/
268
269class VSICurlHandle : public VSIVirtualHandle
270{
271 CPL_DISALLOW_COPY_ASSIGN(VSICurlHandle)
272
273 protected:
274 VSICurlFilesystemHandler* poFS = nullptr;
275
276 bool m_bCached = true;
277
278 FileProp oFileProp{};
279
280 CPLString m_osFilename{}; // e.g "/vsicurl/http://example.com/foo"
281 char* m_pszURL = nullptr; // e.g "http://example.com/foo"
282
283 char **m_papszHTTPOptions = nullptr;
284
285 vsi_l_offset lastDownloadedOffset = VSI_L_OFFSET_MAX;
286 int nBlocksToDownload = 1;
287
288 bool bStopOnInterruptUntilUninstall = false;
289 bool bInterrupted = false;
290 VSICurlReadCbkFunc pfnReadCbk = nullptr;
291 void *pReadCbkUserData = nullptr;
292
293 int m_nMaxRetry = 0;
294 double m_dfRetryDelay = 0.0;
295
296 void DownloadRegionPostProcess( const vsi_l_offset startOffset,
297 const int nBlocks,
298 const char* pBuffer,
299 size_t nSize );
300
301 private:
302
303 vsi_l_offset curOffset = 0;
304
305 bool bEOF = false;
306
307 virtual std::string DownloadRegion(vsi_l_offset startOffset, int nBlocks);
308
309 bool m_bUseHead = false;
310
311 int ReadMultiRangeSingleGet( int nRanges, void ** ppData,
312 const vsi_l_offset* panOffsets,
313 const size_t* panSizes );
314 CPLString GetRedirectURLIfValid(bool& bHasExpired);
315
316 protected:
317 virtual struct curl_slist* GetCurlHeaders( const CPLString& /*osVerb*/,
318 const struct curl_slist* /* psExistingHeaders */)
319 { return nullptr; }
320 virtual bool AllowAutomaticRedirection() { return true; }
321 virtual bool CanRestartOnError( const char*, const char*, bool ) { return false; }
322 virtual bool UseLimitRangeGetInsteadOfHead() { return false; }
323 virtual bool IsDirectoryFromExists( const char* /*pszVerb*/, int /*response_code*/ ) { return false; }
324 virtual void ProcessGetFileSizeResult(const char* /* pszContent */ ) {}
325 void SetURL(const char* pszURL);
326
327 public:
328
329 VSICurlHandle( VSICurlFilesystemHandler* poFS,
330 const char* pszFilename,
331 const char* pszURLIn = nullptr );
332 ~VSICurlHandle() override;
333
334 int Seek( vsi_l_offset nOffset, int nWhence ) override;
335 vsi_l_offset Tell() override;
336 size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
337 int ReadMultiRange( int nRanges, void ** ppData,
338 const vsi_l_offset* panOffsets,
339 const size_t* panSizes ) override;
340 size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
341 int Eof() override;
342 int Flush() override;
343 int Close() override;
344
345 bool IsKnownFileSize() const { return oFileProp.bHasComputedFileSize; }
346 vsi_l_offset GetFileSize() { return GetFileSize(false); }
347 virtual vsi_l_offset GetFileSize( bool bSetError );
348 bool Exists( bool bSetError );
349 bool IsDirectory() const { return oFileProp.bIsDirectory; }
350 time_t GetMTime() const { return oFileProp.mTime; }
351
352 int InstallReadCbk( VSICurlReadCbkFunc pfnReadCbk,
353 void* pfnUserData,
354 int bStopOnInterruptUntilUninstall );
355 int UninstallReadCbk();
356
357 const char *GetURL() const { return m_pszURL; }
358};
359
360/************************************************************************/
361/* IVSIS3LikeFSHandler */
362/************************************************************************/
363
364class IVSIS3LikeFSHandler: public VSICurlFilesystemHandler
365{
366 CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeFSHandler)
367
368 protected:
369 char** GetFileList( const char *pszFilename,
370 int nMaxFiles,
371 bool* pbGotFileList ) override;
372
373 virtual IVSIS3LikeHandleHelper* CreateHandleHelper(
374 const char* pszURI, bool bAllowNoObject) = 0;
375
376 IVSIS3LikeFSHandler() = default;
377
378 public:
379 int Unlink( const char *pszFilename ) override;
380 int Mkdir( const char *pszDirname, long nMode ) override;
381 int Rmdir( const char *pszDirname ) override;
382 int Stat( const char *pszFilename, VSIStatBufL *pStatBuf,
383 int nFlags ) override;
384
385 virtual int DeleteObject( const char *pszFilename );
386
387 virtual const char* GetDebugKey() const = 0;
388
389 virtual void UpdateMapFromHandle(IVSIS3LikeHandleHelper*) {}
390 virtual void UpdateHandleFromMap( IVSIS3LikeHandleHelper * ) {}
391
392 bool Sync( const char* pszSource, const char* pszTarget,
393 const char* const * papszOptions,
394 GDALProgressFunc pProgressFunc,
395 void *pProgressData,
396 char*** ppapszOutputs ) override;
397
398 VSIDIR* OpenDir( const char *pszPath, int nRecurseDepth,
399 const char* const *papszOptions) override;
400};
401
402/************************************************************************/
403/* IVSIS3LikeHandle */
404/************************************************************************/
405
406class IVSIS3LikeHandle: public VSICurlHandle
407{
408 CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeHandle)
409
410 protected:
411 bool UseLimitRangeGetInsteadOfHead() override { return true; }
412 bool IsDirectoryFromExists( const char* pszVerb,
413 int response_code ) override
414 {
415 // A bit dirty, but on S3, a GET on a existing directory returns a 416
416 return response_code == 416 && EQUAL(pszVerb, "GET") &&
417 CPLString(m_pszURL).back() == '/';
418 }
419 void ProcessGetFileSizeResult( const char* pszContent ) override
420 {
421 oFileProp.bIsDirectory = strstr(pszContent, "ListBucketResult") != nullptr;
422 }
423
424 public:
425 IVSIS3LikeHandle( VSICurlFilesystemHandler* poFSIn,
426 const char* pszFilename,
427 const char* pszURLIn = nullptr ) :
428 VSICurlHandle(poFSIn, pszFilename, pszURLIn) {}
429 ~IVSIS3LikeHandle() override {}
430};
431
432/************************************************************************/
433/* VSIS3WriteHandle */
434/************************************************************************/
435
436class VSIS3WriteHandle final : public VSIVirtualHandle
437{
438 CPL_DISALLOW_COPY_ASSIGN(VSIS3WriteHandle)
439
440 IVSIS3LikeFSHandler *m_poFS = nullptr;
441 CPLString m_osFilename{};
442 IVSIS3LikeHandleHelper *m_poS3HandleHelper = nullptr;
443 bool m_bUseChunked = false;
444
445 vsi_l_offset m_nCurOffset = 0;
446 int m_nBufferOff = 0;
447 int m_nBufferSize = 0;
448 int m_nBufferOffReadCallback = 0;
449 bool m_bClosed = false;
450 GByte *m_pabyBuffer = nullptr;
451 CPLString m_osUploadID{};
452 int m_nPartNumber = 0;
453 std::vector<CPLString> m_aosEtags{};
454 CPLString m_osXML{};
455 int m_nOffsetInXML = 0;
456 bool m_bError = false;
457
458 CURLM *m_hCurlMulti = nullptr;
459 CURL *m_hCurl = nullptr;
460 const void *m_pBuffer = nullptr;
461 CPLString m_osCurlErrBuf{};
462 size_t m_nChunkedBufferOff = 0;
463 size_t m_nChunkedBufferSize = 0;
464
465 int m_nMaxRetry = 0;
466 double m_dfRetryDelay = 0.0;
467 WriteFuncStruct m_sWriteFuncHeaderData{};
468
469 static size_t ReadCallBackBuffer( char *buffer, size_t size,
470 size_t nitems, void *instream );
471 bool InitiateMultipartUpload();
472 bool UploadPart();
473 static size_t ReadCallBackXML( char *buffer, size_t size,
474 size_t nitems, void *instream );
475 bool CompleteMultipart();
476 bool AbortMultipart();
477 bool DoSinglePartPUT();
478
479 static size_t ReadCallBackBufferChunked( char *buffer, size_t size,
480 size_t nitems, void *instream );
481 size_t WriteChunked( const void *pBuffer,
482 size_t nSize, size_t nMemb );
483 int FinishChunkedTransfer();
484
485 void InvalidateParentDirectory();
486
487 public:
488 VSIS3WriteHandle( IVSIS3LikeFSHandler* poFS,
489 const char* pszFilename,
490 IVSIS3LikeHandleHelper* poS3HandleHelper,
491 bool bUseChunked );
492 ~VSIS3WriteHandle() override;
493
494 int Seek( vsi_l_offset nOffset, int nWhence ) override;
495 vsi_l_offset Tell() override;
496 size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
497 size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
498 int Eof() override;
499 int Close() override;
500
501 bool IsOK() { return m_bUseChunked || m_pabyBuffer != nullptr; }
502};
503
504/************************************************************************/
505/* VSIAppendWriteHandle */
506/************************************************************************/
507
508class VSIAppendWriteHandle : public VSIVirtualHandle
509{
510 CPL_DISALLOW_COPY_ASSIGN(VSIAppendWriteHandle)
511
512 protected:
513
514 VSICurlFilesystemHandler* m_poFS = nullptr;
515 CPLString m_osFSPrefix{};
516 CPLString m_osFilename{};
517
518 vsi_l_offset m_nCurOffset = 0;
519 int m_nBufferOff = 0;
520 int m_nBufferSize = 0;
521 int m_nBufferOffReadCallback = 0;
522 bool m_bClosed = false;
523 GByte *m_pabyBuffer = nullptr;
524 bool m_bError = false;
525
526 static size_t ReadCallBackBuffer( char *buffer, size_t size,
527 size_t nitems, void *instream );
528 virtual bool Send(bool bIsLastBlock) = 0;
529
530 public:
531 VSIAppendWriteHandle( VSICurlFilesystemHandler* poFS,
532 const char* pszFSPrefix,
533 const char* pszFilename,
534 int nChunkSize );
535 virtual ~VSIAppendWriteHandle();
536
537 int Seek( vsi_l_offset nOffset, int nWhence ) override;
538 vsi_l_offset Tell() override;
539 size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
540 size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
541 int Eof() override;
542 int Close() override;
543
544 bool IsOK() { return m_pabyBuffer != nullptr; }
545};
546
547int VSICURLGetDownloadChunkSize();
548
549void VSICURLInitWriteFuncStruct( WriteFuncStruct *psStruct,
550 VSILFILE *fp,
551 VSICurlReadCbkFunc pfnReadCbk,
552 void *pReadCbkUserData );
553size_t VSICurlHandleWriteFunc( void *buffer, size_t count,
554 size_t nmemb, void *req );
555void MultiPerform(CURLM* hCurlMultiHandle,
556 CURL* hEasyHandle = nullptr);
557void VSICURLResetHeaderAndWriterFunctions(CURL* hCurlHandle);
558
559} // namespace cpl
560
562
563#endif // HAVE_CURL
564
565#endif // CPL_VSIL_CURL_CLASS_H_INCLUDED
String list class designed around our use of C "char**" string lists.
Definition: cpl_string.h:439
Convenient string class based on std::string.
Definition: cpl_string.h:330
Virtual file handle.
Definition: cpl_vsi_virtual.h:56
Core portability definitions for CPL.
#define EQUAL(a, b)
Alias for strcasecmp() == 0.
Definition: cpl_port.h:569
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition: cpl_port.h:997
unsigned char GByte
Unsigned byte type.
Definition: cpl_port.h:215
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition: cpl_port.h:248
Various convenience functions for working with strings and string lists.
#define VSI_L_OFFSET_MAX
Maximum value for a file offset.
Definition: cpl_vsi.h:142
struct VSIDIR VSIDIR
Opaque type for a directory iterator.
Definition: cpl_vsi.h:309
struct VSI_STAT64_T VSIStatBufL
Type for VSIStatL()
Definition: cpl_vsi.h:194
FILE VSILFILE
Opaque type for a FILE that implements the VSIVirtualHandle API.
Definition: cpl_vsi.h:156
GUIntBig vsi_l_offset
Type for a file offset.
Definition: cpl_vsi.h:140

Generated for GDAL by doxygen 1.9.4.