From ae44b1b21a001f1aae37e7baf1d7af4837fa6537 Mon Sep 17 00:00:00 2001 From: Froxcey Date: Wed, 29 May 2024 01:00:03 +0800 Subject: [PATCH 1/5] Add gzip middleware --- src/jetzig/middleware.zig | 1 + src/jetzig/middleware/Gzip.zig | 13 +++++++++++++ 2 files changed, 14 insertions(+) create mode 100644 src/jetzig/middleware/Gzip.zig diff --git a/src/jetzig/middleware.zig b/src/jetzig/middleware.zig index 89b27a6..f392dde 100644 --- a/src/jetzig/middleware.zig +++ b/src/jetzig/middleware.zig @@ -2,6 +2,7 @@ const std = @import("std"); const jetzig = @import("../jetzig.zig"); pub const HtmxMiddleware = @import("middleware/HtmxMiddleware.zig"); +pub const GzipMiddleware = @import("middleware/Gzip.zig"); const RouteOptions = struct { content: ?[]const u8 = null, diff --git a/src/jetzig/middleware/Gzip.zig b/src/jetzig/middleware/Gzip.zig new file mode 100644 index 0000000..9f2bacb --- /dev/null +++ b/src/jetzig/middleware/Gzip.zig @@ -0,0 +1,13 @@ +const std = @import("std"); +const jetzig = @import("jetzig"); + +const err_msg = "Response was not compressed due to error: {s}"; +pub fn beforeResponse(request: *jetzig.http.Request, response: *jetzig.http.Response) !void { + var compressed = std.ArrayList(u8).init(request.allocator); + var reader = std.io.fixedBufferStream(response.content); + std.compress.gzip.compress(reader.reader(), compressed.writer(), .{ .level = .fast }) catch |err| + return request.server.logger.ERROR(err_msg, .{@errorName(err)}); + response.headers.append("Content-Encoding", "gzip") catch |err| + return request.server.logger.ERROR(err_msg, .{@errorName(err)}); + response.content = compressed.items; +} From 4ea023ee5a8b6116f0b1a47aeedf86e7e71393ed Mon Sep 17 00:00:00 2001 From: Froxcey Date: Wed, 29 May 2024 01:01:55 +0800 Subject: [PATCH 2/5] Use gzip in demo --- demo/src/main.zig | 2 ++ 1 file changed, 2 insertions(+) diff --git a/demo/src/main.zig b/demo/src/main.zig index d22356f..535a8cf 100644 --- a/demo/src/main.zig +++ b/demo/src/main.zig @@ -14,6 +14,8 @@ pub const jetzig_options = struct { // htmx middleware skips layouts when `HX-Target` header is present and issues // `HX-Redirect` instead of a regular HTTP redirect when `request.redirect` is called. jetzig.middleware.HtmxMiddleware, + // gzip middleware compresses every request to decrease transfer size + jetzig.middleware.GzipMiddleware, // Demo middleware included with new projects. Remove once you are familiar with Jetzig's // middleware system. @import("app/middleware/DemoMiddleware.zig"), From f538b0ddce1d0eb6fc1a6a673ed14c03ee608ea7 Mon Sep 17 00:00:00 2001 From: Froxcey Date: Wed, 29 May 2024 18:59:05 +0800 Subject: [PATCH 3/5] Fix Compression Middleware Checks "Accept-Encoding" header, add deflate encoding, rename the middleware, and avoid unnecessary encoding. --- demo/src/main.zig | 4 +- src/jetzig/middleware.zig | 2 +- .../middleware/CompressionMiddleware.zig | 72 +++++++++++++++++++ src/jetzig/middleware/Gzip.zig | 13 ---- 4 files changed, 75 insertions(+), 16 deletions(-) create mode 100644 src/jetzig/middleware/CompressionMiddleware.zig delete mode 100644 src/jetzig/middleware/Gzip.zig diff --git a/demo/src/main.zig b/demo/src/main.zig index 535a8cf..2c249bf 100644 --- a/demo/src/main.zig +++ b/demo/src/main.zig @@ -14,8 +14,8 @@ pub const jetzig_options = struct { // htmx middleware skips layouts when `HX-Target` header is present and issues // `HX-Redirect` instead of a regular HTTP redirect when `request.redirect` is called. jetzig.middleware.HtmxMiddleware, - // gzip middleware compresses every request to decrease transfer size - jetzig.middleware.GzipMiddleware, + // Compression middleware compresses every request to decrease transfer size + jetzig.middleware.CompressionMiddleware, // Demo middleware included with new projects. Remove once you are familiar with Jetzig's // middleware system. @import("app/middleware/DemoMiddleware.zig"), diff --git a/src/jetzig/middleware.zig b/src/jetzig/middleware.zig index f392dde..9c01558 100644 --- a/src/jetzig/middleware.zig +++ b/src/jetzig/middleware.zig @@ -2,7 +2,7 @@ const std = @import("std"); const jetzig = @import("../jetzig.zig"); pub const HtmxMiddleware = @import("middleware/HtmxMiddleware.zig"); -pub const GzipMiddleware = @import("middleware/Gzip.zig"); +pub const CompressionMiddleware = @import("middleware/CompressionMiddleware.zig"); const RouteOptions = struct { content: ?[]const u8 = null, diff --git a/src/jetzig/middleware/CompressionMiddleware.zig b/src/jetzig/middleware/CompressionMiddleware.zig new file mode 100644 index 0000000..74406b8 --- /dev/null +++ b/src/jetzig/middleware/CompressionMiddleware.zig @@ -0,0 +1,72 @@ +const std = @import("std"); +const jetzig = @import("jetzig"); + +fn checkType(content_type: []const u8) bool { + const type_list = .{ "text/html", "application/xhtml+xml", "application/xml", "text/css", "text/javascript", "application/json", "application/pdf", "image/svg+xml" }; + inline for (type_list) |content| { + if (std.mem.eql(u8, content_type, content)) return true; + } + return false; +} + +const Encoding = enum { + None, + Gzip, + Deflate, +}; +const err_msg = "Response was not compressed due to error: {s}"; +/// Parse accepted encoding, encode responses if possible, set appropriate headers, and +/// modify the response accordingly to decrease response size +pub fn beforeResponse(request: *jetzig.http.Request, response: *jetzig.http.Response) !void { + // Only some file types need compressions, skip the others + if (!checkType(response.content_type)) return; + + // Find matching encoding + var encoding = Encoding.None; + for (request.headers.getAll("Accept-Encoding")) |encodings| find_encoding: { + var buffer: [64]u8 = undefined; + var encodings_stream = std.io.fixedBufferStream(encodings); + var encodings_reader = encodings_stream.reader(); + while (try encodings_reader.readUntilDelimiterOrEof(&buffer, ',')) |encoding_str| { + const encoding_trimmed = encoding_str[if (encoding_str[0] == ' ') 1 else 0..]; + const encoding_list = .{ "gzip", "deflate" }; + inline for (encoding_list, 0..) |encoding_compare, i| { + if (std.mem.eql(u8, encoding_compare, encoding_trimmed)) { + encoding = @enumFromInt(i + 1); + break :find_encoding; + } + } + std.debug.print("Encoding: {s}\n", .{encoding_str}); + } + } + if (encoding == .None) return; + + // Compress data + var compressed = std.ArrayList(u8).init(request.allocator); + var content_reader = std.io.fixedBufferStream(response.content); + switch (encoding) { + .Gzip => { + std.compress.gzip.compress(content_reader.reader(), compressed.writer(), .{ .level = .fast }) catch |err| + return request.server.logger.ERROR(err_msg, .{@errorName(err)}); + response.headers.append("Content-Encoding", "gzip") catch |err| + return request.server.logger.ERROR(err_msg, .{@errorName(err)}); + }, + .Deflate => { + std.compress.flate.compress(content_reader.reader(), compressed.writer(), .{ .level = .fast }) catch |err| + return request.server.logger.ERROR(err_msg, .{@errorName(err)}); + response.headers.append("Content-Encoding", "deflate") catch |err| + return request.server.logger.ERROR(err_msg, .{@errorName(err)}); + }, + else => { + // The compression is not supported + // TODO: Can add zstd and br in the future, but gzip / deflate + // support through the std is good enough + return; + }, + } + // Make caching work + response.headers.append("Vary", "Accept-Encoding") catch |err| + return request.server.logger.ERROR(err_msg, .{@errorName(err)}); + + response.content = compressed.items; +} diff --git a/src/jetzig/middleware/Gzip.zig b/src/jetzig/middleware/Gzip.zig deleted file mode 100644 index 9f2bacb..0000000 --- a/src/jetzig/middleware/Gzip.zig +++ /dev/null @@ -1,13 +0,0 @@ -const std = @import("std"); -const jetzig = @import("jetzig"); - -const err_msg = "Response was not compressed due to error: {s}"; -pub fn beforeResponse(request: *jetzig.http.Request, response: *jetzig.http.Response) !void { - var compressed = std.ArrayList(u8).init(request.allocator); - var reader = std.io.fixedBufferStream(response.content); - std.compress.gzip.compress(reader.reader(), compressed.writer(), .{ .level = .fast }) catch |err| - return request.server.logger.ERROR(err_msg, .{@errorName(err)}); - response.headers.append("Content-Encoding", "gzip") catch |err| - return request.server.logger.ERROR(err_msg, .{@errorName(err)}); - response.content = compressed.items; -} From 7339641401a0304c204da837e67f05db81e0c9cb Mon Sep 17 00:00:00 2001 From: Bob Farrell Date: Wed, 29 May 2024 19:10:16 +0100 Subject: [PATCH 4/5] Refactoring --- cli/util.zig | 2 +- demo/src/main.zig | 11 +-- src/jetzig/http/Server.zig | 12 +-- src/jetzig/loggers.zig | 6 ++ src/jetzig/loggers/DevelopmentLogger.zig | 13 +++ src/jetzig/loggers/JsonLogger.zig | 4 + .../middleware/CompressionMiddleware.zig | 97 ++++++++----------- src/jetzig/util.zig | 21 ++++ 8 files changed, 92 insertions(+), 74 deletions(-) diff --git a/cli/util.zig b/cli/util.zig index 9ff49d9..002f0b7 100644 --- a/cli/util.zig +++ b/cli/util.zig @@ -82,7 +82,7 @@ fn isPath(dir: std.fs.Dir, sub_path: []const u8, path_type: enum { file, dir }) } // Strip leading and trailing whitespace from a u8 slice. -pub fn strip(input: []const u8) []const u8 { +pub inline fn strip(input: []const u8) []const u8 { return std.mem.trim(u8, input, &std.ascii.whitespace); } diff --git a/demo/src/main.zig b/demo/src/main.zig index 2c249bf..013d928 100644 --- a/demo/src/main.zig +++ b/demo/src/main.zig @@ -11,14 +11,9 @@ pub const jetzig_options = struct { /// Middleware chain. Add any custom middleware here, or use middleware provided in /// `jetzig.middleware` (e.g. `jetzig.middleware.HtmxMiddleware`). pub const middleware: []const type = &.{ - // htmx middleware skips layouts when `HX-Target` header is present and issues - // `HX-Redirect` instead of a regular HTTP redirect when `request.redirect` is called. - jetzig.middleware.HtmxMiddleware, - // Compression middleware compresses every request to decrease transfer size - jetzig.middleware.CompressionMiddleware, - // Demo middleware included with new projects. Remove once you are familiar with Jetzig's - // middleware system. - @import("app/middleware/DemoMiddleware.zig"), + // jetzig.middleware.HtmxMiddleware, + // jetzig.middleware.CompressionMiddleware, + // @import("app/middleware/DemoMiddleware.zig"), }; // Maximum bytes to allow in request body. diff --git a/src/jetzig/http/Server.zig b/src/jetzig/http/Server.zig index faf33b8..6415358 100644 --- a/src/jetzig/http/Server.zig +++ b/src/jetzig/http/Server.zig @@ -398,10 +398,7 @@ fn isBadHttpError(err: anyerror) bool { fn renderInternalServerError(self: *Server, request: *jetzig.http.Request, err: anyerror) !RenderedView { request.response_data.reset(); - try self.logger.ERROR("Encountered Error: {s}", .{@errorName(err)}); - - const stack = @errorReturnTrace(); - if (stack) |capture| try self.logStackTrace(capture, .{ .jetzig = request }); + try self.logger.logError(err); const status = .internal_server_error; return try self.renderError(request, status); @@ -444,12 +441,7 @@ fn renderErrorView( _ = route.render(route.*, request) catch |err| { if (isUnhandledError(err)) return err; - try self.logger.ERROR( - "Unexepected error occurred while rendering error page: {s}", - .{@errorName(err)}, - ); - const stack = @errorReturnTrace(); - if (stack) |capture| try self.logStackTrace(capture, .{ .jetzig = request }); + try self.logger.logError(err); return try renderDefaultError(request, status_code); }; diff --git a/src/jetzig/loggers.zig b/src/jetzig/loggers.zig index 6ce6135..e98f3c8 100644 --- a/src/jetzig/loggers.zig +++ b/src/jetzig/loggers.zig @@ -70,6 +70,12 @@ pub const Logger = union(enum) { } } + pub fn logError(self: *const Logger, err: anyerror) !void { + switch (self.*) { + inline else => |*logger| try logger.logError(err), + } + } + pub fn log( self: *const Logger, comptime level: LogLevel, diff --git a/src/jetzig/loggers/DevelopmentLogger.zig b/src/jetzig/loggers/DevelopmentLogger.zig index 19e6e7b..44d65dd 100644 --- a/src/jetzig/loggers/DevelopmentLogger.zig +++ b/src/jetzig/loggers/DevelopmentLogger.zig @@ -104,6 +104,19 @@ pub fn logRequest(self: DevelopmentLogger, request: *const jetzig.http.Request) }, .stdout); } +pub fn logError(self: *const DevelopmentLogger, err: anyerror) !void { + if (@errorReturnTrace()) |stack| { + try self.log(.ERROR, "\nStack Trace:\n{}", .{stack}); + var buf = std.ArrayList(u8).init(self.allocator); + defer buf.deinit(); + const writer = buf.writer(); + try stack.format("", .{}, writer); + try self.logger.ERROR("{s}\n", .{buf.items}); + } + + try self.log(.ERROR, "Encountered Error: {s}", .{@errorName(err)}); +} + inline fn colorizedLogLevel(comptime level: LogLevel) []const u8 { return switch (level) { .TRACE => jetzig.colors.white(@tagName(level)), diff --git a/src/jetzig/loggers/JsonLogger.zig b/src/jetzig/loggers/JsonLogger.zig index 5beb3e3..eb960d3 100644 --- a/src/jetzig/loggers/JsonLogger.zig +++ b/src/jetzig/loggers/JsonLogger.zig @@ -101,6 +101,10 @@ pub fn logRequest(self: *const JsonLogger, request: *const jetzig.http.Request) try self.log_queue.print("{s}\n", .{stream.getWritten()}, .stdout); } +pub fn logError(self: *const JsonLogger, err: anyerror) !void { + try self.log(.ERROR, "Encountered error: {s}", .{@errorName(err)}); +} + fn getFile(self: JsonLogger, level: LogLevel) std.fs.File { return switch (level) { .TRACE, .DEBUG, .INFO => self.stdout, diff --git a/src/jetzig/middleware/CompressionMiddleware.zig b/src/jetzig/middleware/CompressionMiddleware.zig index 74406b8..5417495 100644 --- a/src/jetzig/middleware/CompressionMiddleware.zig +++ b/src/jetzig/middleware/CompressionMiddleware.zig @@ -1,72 +1,59 @@ const std = @import("std"); -const jetzig = @import("jetzig"); +const jetzig = @import("../../jetzig.zig"); + +fn isCompressable(content_type: []const u8) bool { + const type_list = .{ + "text/html", + "application/xhtml+xml", + "application/xml", + "text/css", + "text/javascript", + "application/json", + "application/pdf", + "image/svg+xml", + }; -fn checkType(content_type: []const u8) bool { - const type_list = .{ "text/html", "application/xhtml+xml", "application/xml", "text/css", "text/javascript", "application/json", "application/pdf", "image/svg+xml" }; inline for (type_list) |content| { if (std.mem.eql(u8, content_type, content)) return true; } return false; } -const Encoding = enum { - None, - Gzip, - Deflate, -}; -const err_msg = "Response was not compressed due to error: {s}"; +const Encoding = enum { gzip, deflate }; /// Parse accepted encoding, encode responses if possible, set appropriate headers, and /// modify the response accordingly to decrease response size pub fn beforeResponse(request: *jetzig.http.Request, response: *jetzig.http.Response) !void { - // Only some file types need compressions, skip the others - if (!checkType(response.content_type)) return; + if (!isCompressable(response.content_type)) return; + const encoding = detectEncoding(request) orelse return; - // Find matching encoding - var encoding = Encoding.None; - for (request.headers.getAll("Accept-Encoding")) |encodings| find_encoding: { - var buffer: [64]u8 = undefined; - var encodings_stream = std.io.fixedBufferStream(encodings); - var encodings_reader = encodings_stream.reader(); - while (try encodings_reader.readUntilDelimiterOrEof(&buffer, ',')) |encoding_str| { - const encoding_trimmed = encoding_str[if (encoding_str[0] == ' ') 1 else 0..]; - const encoding_list = .{ "gzip", "deflate" }; - inline for (encoding_list, 0..) |encoding_compare, i| { - if (std.mem.eql(u8, encoding_compare, encoding_trimmed)) { - encoding = @enumFromInt(i + 1); - break :find_encoding; - } - } - std.debug.print("Encoding: {s}\n", .{encoding_str}); - } - } - if (encoding == .None) return; + const compressed = switch (encoding) { + .gzip => jetzig.util.gzip(request.allocator, response.content, .{}) catch |err| + return request.server.logger.logError(err), + .deflate => jetzig.util.deflate(request.allocator, response.content, .{}) catch |err| + return request.server.logger.logError(err), + }; + + response.headers.append("Content-Encoding", @tagName(encoding)) catch |err| + return request.server.logger.logError(err); - // Compress data - var compressed = std.ArrayList(u8).init(request.allocator); - var content_reader = std.io.fixedBufferStream(response.content); - switch (encoding) { - .Gzip => { - std.compress.gzip.compress(content_reader.reader(), compressed.writer(), .{ .level = .fast }) catch |err| - return request.server.logger.ERROR(err_msg, .{@errorName(err)}); - response.headers.append("Content-Encoding", "gzip") catch |err| - return request.server.logger.ERROR(err_msg, .{@errorName(err)}); - }, - .Deflate => { - std.compress.flate.compress(content_reader.reader(), compressed.writer(), .{ .level = .fast }) catch |err| - return request.server.logger.ERROR(err_msg, .{@errorName(err)}); - response.headers.append("Content-Encoding", "deflate") catch |err| - return request.server.logger.ERROR(err_msg, .{@errorName(err)}); - }, - else => { - // The compression is not supported - // TODO: Can add zstd and br in the future, but gzip / deflate - // support through the std is good enough - return; - }, - } // Make caching work response.headers.append("Vary", "Accept-Encoding") catch |err| - return request.server.logger.ERROR(err_msg, .{@errorName(err)}); + return request.server.logger.logError(err); - response.content = compressed.items; + response.content = compressed; +} + +fn detectEncoding(request: *const jetzig.http.Request) ?Encoding { + for (request.headers.getAll("Accept-Encoding")) |encodings| { + var it = std.mem.tokenizeScalar(u8, encodings, ','); + while (it.next()) |param| { + inline for (@typeInfo(Encoding).Enum.fields) |field| { + if (std.mem.eql(u8, field.name, jetzig.util.strip(param))) { + return std.enums.nameCast(Encoding, field.name); + } + } + } + } + + return null; } diff --git a/src/jetzig/util.zig b/src/jetzig/util.zig index 23a0180..6b05d9f 100644 --- a/src/jetzig/util.zig +++ b/src/jetzig/util.zig @@ -33,6 +33,27 @@ pub fn base64Decode(allocator: std.mem.Allocator, string: []const u8) ![]u8 { return ptr; } +pub fn gzip(allocator: std.mem.Allocator, content: []const u8, options: struct {}) ![]const u8 { + _ = options; // Allow setting compression options later if needed. + var compressed = std.ArrayList(u8).init(allocator); + var content_reader = std.io.fixedBufferStream(content); + try std.compress.gzip.compress(content_reader.reader(), compressed.writer(), .{ .level = .fast }); + return try compressed.toOwnedSlice(); +} + +pub fn deflate(allocator: std.mem.Allocator, content: []const u8, options: struct {}) ![]const u8 { + _ = options; // Allow setting compression options later if needed. + var compressed = std.ArrayList(u8).init(allocator); + var content_reader = std.io.fixedBufferStream(content); + try std.compress.flate.compress(content_reader.reader(), compressed.writer(), .{ .level = .fast }); + return try compressed.toOwnedSlice(); +} + +// Strip leading and trailing whitespace from a u8 slice. +pub inline fn strip(input: []const u8) []const u8 { + return std.mem.trim(u8, input, &std.ascii.whitespace); +} + /// Generate a secure random string of `len` characters (for cryptographic purposes). pub fn generateSecret(allocator: std.mem.Allocator, comptime len: u10) ![]const u8 { const chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; From 3875ffbdfb81cc76fa0d2b3144145a0a5973763a Mon Sep 17 00:00:00 2001 From: Bob Farrell Date: Sat, 1 Jun 2024 17:09:49 +0100 Subject: [PATCH 5/5] Remove allocs in `Headers.getAll()` Provide `Headers.getAllIterator()` to iterate over matching headers, use this to iterate through `Accept-Encoding` headers in CompressionMiddleware. --- src/jetzig/http/Headers.zig | 44 +++++++++++++++++++ .../middleware/CompressionMiddleware.zig | 5 ++- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/src/jetzig/http/Headers.zig b/src/jetzig/http/Headers.zig index 8db8c56..89d3671 100644 --- a/src/jetzig/http/Headers.zig +++ b/src/jetzig/http/Headers.zig @@ -59,6 +59,10 @@ pub fn getFirstValue(self: *const Headers, name: []const u8) ?[]const u8 { return self.get(name); } +pub fn count(self: Headers) usize { + return self.httpz_headers.len; +} + /// Add `name` and `value` to headers. pub fn append(self: *Headers, name: []const u8, value: []const u8) !void { if (self.httpz_headers.len >= self.httpz_headers.keys.len) return error.JetzigTooManyHeaders; @@ -75,6 +79,46 @@ pub fn append(self: *Headers, name: []const u8, value: []const u8) !void { self.httpz_headers.add(header.name, header.value); } +const Iterator = struct { + position: usize = 0, + headers: Headers, + filter_name: ?[]const u8 = null, + + pub fn next(self: *Iterator) ?Header { + const header_count = self.headers.count(); + if (self.position >= header_count) { + return null; + } + const start = self.position; + + var buf: [jetzig.config.get(u16, "max_bytes_header_name")]u8 = undefined; + const filter_name = if (self.filter_name) |name| std.ascii.lowerString(&buf, name) else null; + + for (start..header_count) |index| { + const key = self.headers.httpz_headers.keys[start + index]; + const value = self.headers.httpz_headers.values[start + index]; + self.position += 1; + if (filter_name) |name| { + if (std.mem.eql(u8, name, key)) { + return .{ .name = key, .value = value }; + } + } else { + return .{ .name = key, .value = value }; + } + } + + return null; + } +}; + +pub fn getAllIterator(self: Headers, name: []const u8) Iterator { + return .{ .headers = self, .filter_name = name }; +} + +pub fn iterator(self: Headers) Iterator { + return .{ .headers = self }; +} + test "append (deprecated)" { const allocator = std.testing.allocator; var httpz_headers = try httpz.key_value.KeyValue.init(allocator, 10); diff --git a/src/jetzig/middleware/CompressionMiddleware.zig b/src/jetzig/middleware/CompressionMiddleware.zig index 5417495..038f812 100644 --- a/src/jetzig/middleware/CompressionMiddleware.zig +++ b/src/jetzig/middleware/CompressionMiddleware.zig @@ -44,8 +44,9 @@ pub fn beforeResponse(request: *jetzig.http.Request, response: *jetzig.http.Resp } fn detectEncoding(request: *const jetzig.http.Request) ?Encoding { - for (request.headers.getAll("Accept-Encoding")) |encodings| { - var it = std.mem.tokenizeScalar(u8, encodings, ','); + var headers_it = request.headers.getAllIterator("Accept-Encoding"); + while (headers_it.next()) |header| { + var it = std.mem.tokenizeScalar(u8, header.value, ','); while (it.next()) |param| { inline for (@typeInfo(Encoding).Enum.fields) |field| { if (std.mem.eql(u8, field.name, jetzig.util.strip(param))) {