1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
|
#include "fastfetch.h"
#include "common/library.h"
#include "common/networking.h"
#include "common/strutil.h"
#include "common/debug.h"
#ifdef FF_HAVE_ZLIB
#include <zlib.h>
struct FFZlibLibrary {
FF_LIBRARY_SYMBOL(inflateInit2_)
FF_LIBRARY_SYMBOL(inflate)
FF_LIBRARY_SYMBOL(inflateEnd)
bool inited;
} zlibData;
const char* ffNetworkingLoadZlibLibrary(void) {
if (!zlibData.inited) {
zlibData.inited = true;
FF_LIBRARY_LOAD_MESSAGE(zlib,
#ifdef _WIN32
"zlib1"
#else
"libz"
#endif
FF_LIBRARY_EXTENSION,
2)
FF_LIBRARY_LOAD_SYMBOL_VAR_MESSAGE(zlib, zlibData, inflateInit2_)
FF_LIBRARY_LOAD_SYMBOL_VAR_MESSAGE(zlib, zlibData, inflate)
FF_LIBRARY_LOAD_SYMBOL_VAR_MESSAGE(zlib, zlibData, inflateEnd)
zlib = NULL; // don't auto dlclose
}
return zlibData.ffinflateEnd == NULL ? "Failed to load libz" : NULL;
}
// Try to pre-read gzip header to determine uncompressed size
static uint32_t guessGzipOutputSize(const void* data, uint32_t dataSize) {
// gzip file format: http://www.zlib.org/rfc-gzip.html
if (dataSize < 10 || ((const uint8_t*) data)[0] != 0x1f || ((const uint8_t*) data)[1] != 0x8b) {
return 0;
}
// Uncompressed size in gzip format is stored in the last 4 bytes, but only valid if data is less than 4GB
if (dataSize > 18) {
// Get ISIZE value from the end of file (little endian)
const uint8_t* tail = (const uint8_t*) data + dataSize - 4;
uint32_t uncompressedSize = (uint32_t) tail[0] | ((uint32_t) tail[1] << 8u) | ((uint32_t) tail[2] << 16u) | ((uint32_t) tail[3] << 24u);
// For valid gzip files, this value is the length of the uncompressed data modulo 2^32
if (uncompressedSize > 0) {
FF_DEBUG("Read uncompressed size from GZIP trailer: %u bytes", uncompressedSize);
// Add some margin to the estimated size for safety
return uncompressedSize + 64;
}
}
// If unable to get size from trailer or size is 0, use estimated value
// Typically, text data compression ratio is between 3-5x, we use the larger value
uint32_t estimatedSize = dataSize * 5;
FF_DEBUG("Unable to read exact uncompressed size, estimated as 5x of compressed data: %u bytes", estimatedSize);
return estimatedSize;
}
// Decompress gzip content
bool ffNetworkingDecompressGzip(FFstrbuf* buffer, char* headerEnd) {
assert(headerEnd != NULL && *headerEnd == '\r');
// Calculate header size
uint32_t headerSize = (uint32_t) (headerEnd - buffer->chars);
*headerEnd = '\0'; // Replace delimiter with null character for easier processing
// Ensure Content-Encoding is in response headers, not in response body
bool hasGzip = strcasestr(buffer->chars, "\nContent-Encoding: gzip") != NULL;
*headerEnd = '\r'; // Restore delimiter
if (!hasGzip) {
FF_DEBUG("No gzip compressed content detected, skipping decompression");
return true;
}
FF_DEBUG("Gzip compressed content detected, preparing for decompression");
const char* bodyStart = headerEnd + 4; // Skip delimiter
if (buffer->length <= headerSize + 4) {
// No content to decompress
FF_DEBUG("Compressed content size is 0, skipping decompression");
return true;
}
// Calculate compressed content size
uint32_t compressedSize = buffer->length - headerSize - 4;
// Check if content is actually in gzip format (gzip header magic is 0x1f 0x8b)
if (compressedSize < 2 || (uint8_t) bodyStart[0] != 0x1f || (uint8_t) bodyStart[1] != 0x8b) {
FF_DEBUG("Content is not valid gzip format, skipping decompression");
return false;
}
// Predict uncompressed size
uint32_t estimatedSize = guessGzipOutputSize(bodyStart, compressedSize);
// Create decompression buffer with estimated size
FF_STRBUF_AUTO_DESTROY decompressedBuffer = ffStrbufCreateA(estimatedSize > 0 ? estimatedSize : compressedSize * 5);
FF_DEBUG("Created decompression buffer: %u bytes", decompressedBuffer.allocated);
// Initialize decompression
z_stream zs = {
.zalloc = Z_NULL,
.zfree = Z_NULL,
.opaque = Z_NULL,
.avail_in = (uInt) compressedSize,
.next_in = (Bytef*) bodyStart,
.avail_out = (uInt) ffStrbufGetFree(&decompressedBuffer),
.next_out = (Bytef*) decompressedBuffer.chars,
};
// Initialize decompression engine
if (zlibData.ffinflateInit2_(&zs, 16 + MAX_WBITS, ZLIB_VERSION, (int) sizeof(z_stream)) != Z_OK) {
FF_DEBUG("Failed to initialize decompression engine");
return false;
}
uInt availableOut = zs.avail_out;
// Perform decompression
int result = zlibData.ffinflate(&zs, Z_FINISH);
// If output buffer is insufficient, try to extend buffer
while (result == Z_BUF_ERROR || (result != Z_STREAM_END && zs.avail_out == 0)) {
FF_DEBUG("Output buffer insufficient, trying to extend");
// Save already decompressed data amount
uint32_t alreadyDecompressed = (uint32_t) (availableOut - zs.avail_out);
decompressedBuffer.length += alreadyDecompressed;
decompressedBuffer.chars[decompressedBuffer.length] = '\0';
ffStrbufEnsureFree(&decompressedBuffer, decompressedBuffer.length / 2);
// Set output parameters to point to new buffer
zs.avail_out = (uInt) ffStrbufGetFree(&decompressedBuffer);
zs.next_out = (Bytef*) (decompressedBuffer.chars + decompressedBuffer.length);
availableOut = zs.avail_out;
// Decompress again
result = zlibData.ffinflate(&zs, Z_FINISH);
}
// Check for decompression errors before using result
if (result != Z_STREAM_END) {
FF_DEBUG("Decompression failed with zlib error: %d", result);
zlibData.ffinflateEnd(&zs);
return false;
}
zlibData.ffinflateEnd(&zs);
// Calculate decompressed size (from the last inflate call)
uint32_t decompressedSize = (uint32_t) (availableOut - zs.avail_out);
decompressedBuffer.length += decompressedSize;
decompressedBuffer.chars[decompressedBuffer.length] = '\0';
FF_DEBUG("Successfully decompressed %u bytes compressed data to %u bytes", compressedSize, decompressedBuffer.length);
// Modify Content-Length header and remove Content-Encoding header
// Use decompressedBuffer.length (total) not decompressedSize (last chunk only)
FF_STRBUF_AUTO_DESTROY newBuffer = ffStrbufCreateA(headerSize + decompressedBuffer.length + 64);
char* line = NULL;
size_t len = 0;
while (ffStrbufGetline(&line, &len, buffer)) {
if (ffStrStartsWithIgnCase(line, "Content-Encoding:")) {
continue;
} else if (ffStrStartsWithIgnCase(line, "Content-Length:")) {
ffStrbufAppendF(&newBuffer, "Content-Length: %u\r\n", decompressedBuffer.length);
continue;
} else if (line[0] == '\r') {
ffStrbufAppendS(&newBuffer, "\r\n");
ffStrbufGetlineRestore(&line, &len, buffer);
break;
}
ffStrbufAppendS(&newBuffer, line); // Including the trailing \r
ffStrbufAppendC(&newBuffer, '\n');
}
ffStrbufAppend(&newBuffer, &decompressedBuffer);
ffStrbufDestroy(buffer);
ffStrbufInitMove(buffer, &newBuffer);
return true;
}
#endif // FF_HAVE_ZLIB
|
