Feature/availability loader and lazy page sizes (#185)

* Fix opening a document when a single page cannot be resolved

PdfDocument measured every page in its constructor, so one page pdfium could
not resolve failed the whole document, even for callers that only wanted the
page count or a different page. PageSizes is now measured per page on first
read and the count comes from FPDF_GetPageCount, which resolves no pages.
GetPageSizes still measures every page and still reports an unreadable one.

* Reach pages of linearized documents through FPDFAvail

Where walking /Pages dead-ends, pages present in the file report as missing.
FPDFAvail_* fills page_list_ from the linearization hint tables instead, so
GetPageDictionary resolves them without traversing: on a 3467 page document
the walk reaches 46 pages and FPDFAvail all of them. PdfFile reopens that way
once per document, only after a page has failed and only when
FPDFAvail_IsLinearized does not say no, so documents that work today keep
their path and a failed reopen leaves the usual page error.

Also stops PdfException.CreateException(SUCCESS) being dereferenced with "!"
when pdfium refuses a page without recording an error.

* Use the availability API for all documents; add unit tests for Stream related issues

* API cleanup and restricted support of >4 GiB PDF files

* Add tests for the 4 GiB PDF file size limit

---------

Co-authored-by: David Sungaila <david.sungaila@arcor.de>

Author: JessThrysoee

src/Directory.Packages.props

@@ -19,6 +19,7 @@
     <PackageVersion Include="Microsoft.Maui.Controls" Version="10.0.90" />
     <PackageVersion Include="Microsoft.Maui.Controls.Compatibility" Version="10.0.90" />
     <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="18.8.1" />
+    <PackageVersion Include="Microsoft.Windows.CsWin32" Version="0.3.298" />
     <PackageVersion Include="MSTest.TestAdapter" Version="4.3.2" />
     <PackageVersion Include="MSTest.TestFramework" Version="4.3.2" />
     <PackageVersion Include="PatrickJahr.Blazor.FileHandling" Version="1.0.0" />

src/PDFtoImage/Conversion.Stream.cs

@@ -7,6 +7,7 @@
 using System.Linq;
 using System.Runtime.CompilerServices;
 using System.Threading;
+using System.Threading.Tasks;
 
 namespace PDFtoImage
 {
@@ -91,10 +92,10 @@ public static IEnumerable<SKBitmap> ToImages(Stream pdfStream, IEnumerable<int>
 
             var pageCount = pdfDocument.PageSizes.Count;
 
-            if (validatedPages.Any(p => p >= pageCount))
+            if (validatedPages.Any(p => p < 0 || p >= pageCount))
                 throw new ArgumentOutOfRangeException(nameof(pages), $"The page numbers must be between 0 and {pageCount - 1}. The PDF has {pageCount} pages in total.");
 
-            foreach (var bitmap in ToImagesImpl(pdfStream, leaveOpen, password, options, validatedPages))
+            foreach (var bitmap in ToImagesImpl(pdfDocument, options, validatedPages))
             {
                 yield return bitmap;
             }
@@ -259,7 +260,7 @@ public static IEnumerable<SKBitmap> ToImages(Stream pdfStream, Range pages, bool
 
             var pageNumbers = Enumerable.Range(offset, length);
 
-            foreach (var bitmap in ToImagesImpl(pdfStream, leaveOpen, password, options, pageNumbers))
+            foreach (var bitmap in ToImagesImpl(pdfDocument, options, pageNumbers))
             {
                 yield return bitmap;
             }
@@ -282,7 +283,7 @@ public static async IAsyncEnumerable<SKBitmap> ToImagesAsync(Stream pdfStream, R
                 throw new ArgumentNullException(nameof(pdfStream));
 
             // Stream -> Internals.PdfDocument
-            using var pdfDocument = PdfDocument.Load(pdfStream, password, !leaveOpen);
+            using var pdfDocument = await Task.Run(() => PdfDocument.Load(pdfStream, password, !leaveOpen), cancellationToken);
 
             var pageCount = pdfDocument.PageSizes.Count;
             var (offset, length) = pages.GetOffsetAndLength(pageCount);
@@ -292,7 +293,7 @@ public static async IAsyncEnumerable<SKBitmap> ToImagesAsync(Stream pdfStream, R
 
             var pageNumbers = Enumerable.Range(offset, length);
 
-            await foreach (var bitmap in ToImagesImplAsync(pdfStream, leaveOpen, password, options, pageNumbers, cancellationToken))
+            await foreach (var bitmap in ToImagesImplAsync(pdfDocument, options, pageNumbers, cancellationToken))
             {
                 yield return bitmap;
             }
@@ -319,14 +320,14 @@ public static async IAsyncEnumerable<SKBitmap> ToImagesAsync(Stream pdfStream, I
             var validatedPages = pages.ToArray();
 
             // Stream -> Internals.PdfDocument
-            using var pdfDocument = PdfDocument.Load(pdfStream, password, !leaveOpen);
+            using var pdfDocument = await Task.Run(() => PdfDocument.Load(pdfStream, password, !leaveOpen), cancellationToken);
 
             var pageCount = pdfDocument.PageSizes.Count;
 
-            if (validatedPages.Any(p => p >= pageCount))
+            if (validatedPages.Any(p => p < 0 || p >= pageCount))
                 throw new ArgumentOutOfRangeException(nameof(pages), $"The page numbers must be between 0 and {pageCount - 1}. The PDF has {pageCount} pages in total.");
 
-            await foreach (var bitmap in ToImagesImplAsync(pdfStream, leaveOpen, password, options, validatedPages, cancellationToken))
+            await foreach (var bitmap in ToImagesImplAsync(pdfDocument, options, validatedPages, cancellationToken))
             {
                 yield return bitmap;
             }

src/PDFtoImage/Conversion.cs

@@ -62,12 +62,20 @@ internal static async IAsyncEnumerable<SKBitmap> ToImagesImplAsync(Stream pdfStr
             if (pdfStream == null)
                 throw new ArgumentNullException(nameof(pdfStream));
 
-            if (options == default)
-                options = new();
-
             // Stream -> Internals.PdfDocument
             using var pdfDocument = await Task.Run(() => PdfDocument.Load(pdfStream, password, !leaveOpen), cancellationToken);
 
+            await foreach (var bitmap in ToImagesImplAsync(pdfDocument, options, pages, cancellationToken))
+            {
+                yield return bitmap;
+            }
+        }
+
+        internal static async IAsyncEnumerable<SKBitmap> ToImagesImplAsync(PdfDocument pdfDocument, RenderOptions options, IEnumerable<int>? pages, [EnumeratorCancellation] CancellationToken cancellationToken = default)
+        {
+            if (options == default)
+                options = new();
+
             pages ??= Enumerable.Range(0, pdfDocument.PageSizes.Count);
 
             foreach (var page in pages)

src/PDFtoImage/Internals/NativeMethods.DllImport.cs

@@ -10,39 +10,31 @@ namespace PDFtoImage.Internals
 {
     internal static partial class NativeMethods
     {
-        public static void SetFormFieldHighlightColor(IntPtr hHandle, int fieldType, uint color)
+        public static bool Bitmap_FillRect(IntPtr bitmapHandle, int left, int top, int width, int height, uint color)
         {
             lock (LockString)
             {
-                Imports.FPDF_SetFormFieldHighlightColor(hHandle, fieldType, color);
+                return Imports.FPDFBitmap_FillRect(bitmapHandle, left, top, width, height, color) != 0;
             }
         }
 
-        public static bool Bitmap_FillRect(IntPtr bitmapHandle, int left, int top, int width, int height, uint color)
+        public static bool GetPageSizeByIndex(IntPtr document, int page_index, out double width, out double height)
         {
             lock (LockString)
             {
-                return Imports.FPDFBitmap_FillRect(bitmapHandle, left, top, width, height, color) != 0;
+                return Imports.FPDF_GetPageSizeByIndex(document, page_index, out width, out height) != 0;
             }
         }
 
-        public static bool GetPageSizeByIndex(IntPtr document, int page_index, out double width, out double height)
+        public static FPDF_ERR GetLastError()
         {
             lock (LockString)
             {
-                return Imports.FPDF_GetPageSizeByIndex(document, page_index, out width, out height) != 0;
+                return (FPDF_ERR)Imports.FPDF_GetLastError();
             }
         }
 
-        /// <summary>
-        /// Opens a document using a .NET Stream. Allows opening huge
-        /// PDFs without loading them into memory first.
-        /// </summary>
-        /// <param name="input">The input Stream. Don't dispose prior to closing the pdf.</param>
-        /// <param name="password">Password, if the PDF is protected. Can be null.</param>
-        /// <param name="id">Retrieves an IntPtr to the COM object for the Stream. The caller must release this with Marshal.Release prior to Disposing the Stream.</param>
-        /// <returns>An IntPtr to the FPDF_DOCUMENT object.</returns>
-        public unsafe static IntPtr LoadCustomDocument(Stream input, string? password, int id)
+        private unsafe static IntPtr CreateAvailFileAccessState(Stream input, int id)
         {
 #if BROWSER
             delegate* unmanaged[Cdecl]<IntPtr, uint, IntPtr, uint, int> getBlock = &FPDF_GetBlock;
@@ -52,38 +44,71 @@ public unsafe static IntPtr LoadCustomDocument(Stream input, string? password, i
             var access = new FPDF_FILEACCESS((uint)input.Length, getBlock, (IntPtr)id);
 #endif
 
-            var size = Marshal.SizeOf<FPDF_FILEACCESS>();
-            var ptr = Marshal.AllocHGlobal(size);
-            Marshal.StructureToPtr(access, ptr, false);
-
-            byte[]? passwordBytes = password != null
-               ? Encoding.UTF8.GetBytes(password + '\0')
-               : null;
+            var fileAccessState = Marshal.AllocHGlobal(Marshal.SizeOf<FPDF_FILEACCESS>());
 
             try
             {
-                fixed (byte* passwordPointer = passwordBytes)
-                {
-                    lock (LockString)
-                    {
-                        return Imports.FPDF_LoadCustomDocument(ptr, (IntPtr)passwordPointer);
-                    }
-                }
+                Marshal.StructureToPtr(access, fileAccessState, false);
+                return fileAccessState;
             }
-            finally
+            catch
             {
-                Marshal.FreeHGlobal(ptr);
+                Marshal.FreeHGlobal(fileAccessState);
+                throw;
             }
         }
 
-        public static FPDF_ERR GetLastError()
+        private unsafe static IntPtr Avail_GetDocumentCore(IntPtr avail, string? password)
         {
-            lock (LockString)
+            byte[]? passwordBytes = password != null
+               ? Encoding.UTF8.GetBytes(password + '\0')
+               : null;
+
+            fixed (byte* passwordPointer = passwordBytes)
             {
-                return (FPDF_ERR)Imports.FPDF_GetLastError();
+                return Imports.FPDFAvail_GetDocument(avail, (IntPtr)passwordPointer);
             }
         }
 
+        private unsafe static IntPtr GetIsDataAvailCallbackPointer()
+        {
+#if BROWSER
+            delegate* unmanaged[Cdecl]<IntPtr, UIntPtr, UIntPtr, int> callback = &FX_IsDataAvail;
+            return (IntPtr)callback;
+#else
+            return Marshal.GetFunctionPointerForDelegate(_isDataAvailDelegate);
+#endif
+        }
+
+        private unsafe static IntPtr GetAddSegmentCallbackPointer()
+        {
+#if BROWSER
+            delegate* unmanaged[Cdecl]<IntPtr, UIntPtr, UIntPtr, void> callback = &FX_AddSegment;
+            return (IntPtr)callback;
+#else
+            return Marshal.GetFunctionPointerForDelegate(_addSegmentDelegate);
+#endif
+        }
+
+        // PDFtoImage gives PDFium a complete seekable stream, matching pdfium_test's local-file
+        // availability provider: all requested byte ranges are reported as present and download
+        // hints are intentionally ignored.
+#if BROWSER
+[UnmanagedCallersOnly(CallConvs = [typeof(CallConvCdecl)])]
+#else
+        // needed for Unity IL2CPP compilation
+        [AOT.MonoPInvokeCallback(typeof(FX_IsDataAvailDelegate))]
+#endif
+        private static int FX_IsDataAvail(IntPtr param, UIntPtr offset, UIntPtr size) => 1;
+
+#if BROWSER
+[UnmanagedCallersOnly(CallConvs = [typeof(CallConvCdecl)])]
+#else
+        // needed for Unity IL2CPP compilation
+        [AOT.MonoPInvokeCallback(typeof(FX_AddSegmentDelegate))]
+#endif
+        private static void FX_AddSegment(IntPtr param, UIntPtr offset, UIntPtr size) { }
+
 #if BROWSER
 [UnmanagedCallersOnly(CallConvs = [typeof(CallConvCdecl)])]
 #else
@@ -168,12 +193,6 @@ private static partial class Imports
             [DllImport("pdfium", CallingConvention = CallingConvention.Cdecl)]
             public static extern int FPDF_GetPageCount(IntPtr document);
 
-            [DllImport("pdfium", CallingConvention = CallingConvention.Cdecl)]
-            public static extern void FPDF_SetFormFieldHighlightColor(IntPtr hHandle, int fieldType, uint color);
-
-            [DllImport("pdfium", CallingConvention = CallingConvention.Cdecl)]
-            public static extern void FPDF_SetFormFieldHighlightAlpha(IntPtr hHandle, byte alpha);
-
             [DllImport("pdfium", CallingConvention = CallingConvention.Cdecl)]
             public static extern IntPtr FPDF_LoadPage(IntPtr document, int page_index);
 
@@ -216,9 +235,24 @@ private static partial class Imports
             [DllImport("pdfium", CallingConvention = CallingConvention.Cdecl)]
             public static extern void FPDF_RemoveFormFieldHighlight(IntPtr form);
 
+            [DllImport("pdfium", CallingConvention = CallingConvention.Cdecl)]
+            public static extern IntPtr FPDFAvail_Create(IntPtr file_avail, IntPtr file);
+
+            [DllImport("pdfium", CallingConvention = CallingConvention.Cdecl)]
+            public static extern void FPDFAvail_Destroy(IntPtr avail);
+
+            [DllImport("pdfium", CallingConvention = CallingConvention.Cdecl)]
+            public static extern int FPDFAvail_IsDocAvail(IntPtr avail, IntPtr hints);
+
+            [DllImport("pdfium", CallingConvention = CallingConvention.Cdecl)]
+            public static extern int FPDFAvail_IsPageAvail(IntPtr avail, int page_index, IntPtr hints);
+
+            [DllImport("pdfium", CallingConvention = CallingConvention.Cdecl)]
+            public static extern int FPDFAvail_IsFormAvail(IntPtr avail, IntPtr hints);
+
             [DllImport("pdfium", CallingConvention = CallingConvention.Cdecl)]
             [System.Diagnostics.CodeAnalysis.SuppressMessage("Globalization", "CA2101")]
-            public static extern IntPtr FPDF_LoadCustomDocument(IntPtr access, IntPtr password);
+            public static extern IntPtr FPDFAvail_GetDocument(IntPtr avail, IntPtr password);
 
             [DllImport("pdfium", CallingConvention = CallingConvention.Cdecl)]
             public static extern IntPtr FPDFDOC_InitFormFillEnvironment(IntPtr document, IntPtr formInfo);
@@ -232,8 +266,18 @@ private static partial class Imports
         [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
         private delegate int FPDF_GetBlockDelegate(IntPtr param, uint position, IntPtr buffer, uint size);
 
+        [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
+        private delegate int FX_IsDataAvailDelegate(IntPtr param, UIntPtr offset, UIntPtr size);
+
+        [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
+        private delegate void FX_AddSegmentDelegate(IntPtr param, UIntPtr offset, UIntPtr size);
+
 #if !BROWSER
         private static readonly FPDF_GetBlockDelegate _getBlockDelegate = FPDF_GetBlock;
+
+        private static readonly FX_IsDataAvailDelegate _isDataAvailDelegate = FX_IsDataAvail;
+
+        private static readonly FX_AddSegmentDelegate _addSegmentDelegate = FX_AddSegment;
 #endif
 
         [StructLayout(LayoutKind.Sequential)]
@@ -243,6 +287,7 @@ public readonly struct FPDF_FILEACCESS(uint m_FileLen, IntPtr m_GetBlock, IntPtr
             private readonly IntPtr m_GetBlock = m_GetBlock;
             private readonly IntPtr m_Param = m_Param;
         }
+
     }
 }
 #endif