Giter Club home page Giter Club logo

Comments (4)

nigeltao avatar nigeltao commented on May 24, 2024

Because the pixel swizzler (converting from one pixel format to another) only supports a limited number of destination formats.

wuffs/internal/cgen/base/pixconv-submodule-regular.c

Lines 5153 to 5225 in 49f50cc

switch (src_pixfmt.repr) {
case WUFFS_BASE__PIXEL_FORMAT__Y:
func = wuffs_base__pixel_swizzler__prepare__y(p, dst_pixfmt, dst_palette,
src_palette, blend);
break;
case WUFFS_BASE__PIXEL_FORMAT__Y_16BE:
func = wuffs_base__pixel_swizzler__prepare__y_16be(
p, dst_pixfmt, dst_palette, src_palette, blend);
break;
case WUFFS_BASE__PIXEL_FORMAT__INDEXED__BGRA_NONPREMUL:
func = wuffs_base__pixel_swizzler__prepare__indexed__bgra_nonpremul(
p, dst_pixfmt, dst_palette, src_palette, blend);
break;
case WUFFS_BASE__PIXEL_FORMAT__INDEXED__BGRA_BINARY:
func = wuffs_base__pixel_swizzler__prepare__indexed__bgra_binary(
p, dst_pixfmt, dst_palette, src_palette, blend);
break;
case WUFFS_BASE__PIXEL_FORMAT__BGR_565:
func = wuffs_base__pixel_swizzler__prepare__bgr_565(
p, dst_pixfmt, dst_palette, src_palette, blend);
break;
case WUFFS_BASE__PIXEL_FORMAT__BGR:
func = wuffs_base__pixel_swizzler__prepare__bgr(
p, dst_pixfmt, dst_palette, src_palette, blend);
break;
case WUFFS_BASE__PIXEL_FORMAT__BGRA_NONPREMUL:
func = wuffs_base__pixel_swizzler__prepare__bgra_nonpremul(
p, dst_pixfmt, dst_palette, src_palette, blend);
break;
case WUFFS_BASE__PIXEL_FORMAT__BGRA_NONPREMUL_4X16LE:
func = wuffs_base__pixel_swizzler__prepare__bgra_nonpremul_4x16le(
p, dst_pixfmt, dst_palette, src_palette, blend);
break;
case WUFFS_BASE__PIXEL_FORMAT__BGRA_PREMUL:
func = wuffs_base__pixel_swizzler__prepare__bgra_premul(
p, dst_pixfmt, dst_palette, src_palette, blend);
break;
case WUFFS_BASE__PIXEL_FORMAT__BGRX:
func = wuffs_base__pixel_swizzler__prepare__bgrx(
p, dst_pixfmt, dst_palette, src_palette, blend);
break;
case WUFFS_BASE__PIXEL_FORMAT__RGB:
func = wuffs_base__pixel_swizzler__prepare__rgb(
p, dst_pixfmt, dst_palette, src_palette, blend);
break;
case WUFFS_BASE__PIXEL_FORMAT__RGBA_NONPREMUL:
func = wuffs_base__pixel_swizzler__prepare__rgba_nonpremul(
p, dst_pixfmt, dst_palette, src_palette, blend);
break;
case WUFFS_BASE__PIXEL_FORMAT__RGBA_PREMUL:
func = wuffs_base__pixel_swizzler__prepare__rgba_premul(
p, dst_pixfmt, dst_palette, src_palette, blend);
break;
}
p->private_impl.func = func;
p->private_impl.transparent_black_func = transparent_black_func;
p->private_impl.dst_pixfmt_bytes_per_pixel = dst_pixfmt_bits_per_pixel / 8;
p->private_impl.src_pixfmt_bytes_per_pixel = src_pixfmt_bits_per_pixel / 8;
return wuffs_base__make_status(
func ? NULL : wuffs_base__error__unsupported_pixel_swizzler_option);

is the relevant code. Note that the switch statement looks at the source pixel format and you have to drill into the functions (e.g.

wuffs/internal/cgen/base/pixconv-submodule-regular.c

Lines 4694 to 4771 in 49f50cc

switch (dst_pixfmt.repr) {
case WUFFS_BASE__PIXEL_FORMAT__BGR_565:
switch (blend) {
case WUFFS_BASE__PIXEL_BLEND__SRC:
return wuffs_base__pixel_swizzler__bgr_565__bgra_nonpremul_4x16le__src;
case WUFFS_BASE__PIXEL_BLEND__SRC_OVER:
return wuffs_base__pixel_swizzler__bgr_565__bgra_nonpremul_4x16le__src_over;
}
return NULL;
case WUFFS_BASE__PIXEL_FORMAT__BGR:
switch (blend) {
case WUFFS_BASE__PIXEL_BLEND__SRC:
return wuffs_base__pixel_swizzler__bgr__bgra_nonpremul_4x16le__src;
case WUFFS_BASE__PIXEL_BLEND__SRC_OVER:
return wuffs_base__pixel_swizzler__bgr__bgra_nonpremul_4x16le__src_over;
}
return NULL;
case WUFFS_BASE__PIXEL_FORMAT__BGRA_NONPREMUL:
switch (blend) {
case WUFFS_BASE__PIXEL_BLEND__SRC:
return wuffs_base__pixel_swizzler__bgra_nonpremul__bgra_nonpremul_4x16le__src;
case WUFFS_BASE__PIXEL_BLEND__SRC_OVER:
return wuffs_base__pixel_swizzler__bgra_nonpremul__bgra_nonpremul_4x16le__src_over;
}
return NULL;
case WUFFS_BASE__PIXEL_FORMAT__BGRA_NONPREMUL_4X16LE:
switch (blend) {
case WUFFS_BASE__PIXEL_BLEND__SRC:
return wuffs_base__pixel_swizzler__copy_8_8;
case WUFFS_BASE__PIXEL_BLEND__SRC_OVER:
return wuffs_base__pixel_swizzler__bgra_nonpremul_4x16le__bgra_nonpremul_4x16le__src_over;
}
return NULL;
case WUFFS_BASE__PIXEL_FORMAT__BGRA_PREMUL:
switch (blend) {
case WUFFS_BASE__PIXEL_BLEND__SRC:
return wuffs_base__pixel_swizzler__bgra_premul__bgra_nonpremul_4x16le__src;
case WUFFS_BASE__PIXEL_BLEND__SRC_OVER:
return wuffs_base__pixel_swizzler__bgra_premul__bgra_nonpremul_4x16le__src_over;
}
return NULL;
case WUFFS_BASE__PIXEL_FORMAT__BGRA_BINARY:
case WUFFS_BASE__PIXEL_FORMAT__BGRX:
// TODO.
break;
case WUFFS_BASE__PIXEL_FORMAT__RGB:
// TODO.
break;
case WUFFS_BASE__PIXEL_FORMAT__RGBA_NONPREMUL:
switch (blend) {
case WUFFS_BASE__PIXEL_BLEND__SRC:
return wuffs_base__pixel_swizzler__rgba_nonpremul__bgra_nonpremul_4x16le__src;
case WUFFS_BASE__PIXEL_BLEND__SRC_OVER:
return wuffs_base__pixel_swizzler__rgba_nonpremul__bgra_nonpremul_4x16le__src_over;
}
break;
case WUFFS_BASE__PIXEL_FORMAT__RGBA_PREMUL:
switch (blend) {
case WUFFS_BASE__PIXEL_BLEND__SRC:
return wuffs_base__pixel_swizzler__bgra_premul__rgba_nonpremul_4x16le__src;
case WUFFS_BASE__PIXEL_BLEND__SRC_OVER:
return wuffs_base__pixel_swizzler__bgra_premul__rgba_nonpremul_4x16le__src_over;
}
return NULL;
case WUFFS_BASE__PIXEL_FORMAT__RGBA_BINARY:
case WUFFS_BASE__PIXEL_FORMAT__RGBX:
// TODO.
break;
}
) to see supported destination pixel formats.

For N pixel formats, there are N*N different (source, destination) combinations. It would be a lot of code (not just to write, but there's also a binary-size cost) to support all of the combinations. Instead, we support a smaller subset: supported source pixel formats are based on what's 'natural' for the image file formats (JPEG, PNG, etc) and supported destination pixel formats are based on what's commonly used.

Is there a particular destination format that you'd like added, or are you just curious?

from wuffs.

dev0x13 avatar dev0x13 commented on May 24, 2024

Thank you very much for such a detailed response! I was initially interested in WUFFS_BASE__PIXEL_FORMAT__RGB destination pixel format, which appeared to be working fine for each input image from my PNG dataset, so I was curious why the auxiliary API doesn't cover this format too despite it's being quite 'natural' (for historical reasons) when it comes to computer vision models, especially neural networks. But now I see that WUFFS_BASE__PIXEL_FORMAT__RGB has somewhat limited support (lots of TODOs across the swizzler dispatching code), so it's actually might be safer to disable it at a higher level. However, it seems to me like for wuffs_aux::DecodeImage code it might make sense to just pass through the destination pixel format parameter and let the actual decoder itself fail later while selecting a pixel swizzler. Or am I getting this wrong?
Anyway, thanks again and I'm closing the issue since the initial question was answered.

from wuffs.

nigeltao avatar nigeltao commented on May 24, 2024

I was initially interested in WUFFS_BASE__PIXEL_FORMAT__RGB destination pixel format, which appeared to be working fine for each input image from my PNG dataset, so I was curious why the auxiliary API doesn't cover this format too

WUFFS_BASE__PIXEL_FORMAT__RGB works for most opaque (no alpha channel) PNG images because that's "the image file's natural pixel format" from

wuffs/internal/cgen/auxiliary/image.hh

Lines 129 to 131 in 4dfd709

// or return image_config.pixcfg.pixel_format(). The latter means to use the
// image file's natural pixel format. For example, GIF images' natural pixel
// format is an indexed one.

In terms of the bytes in the file format, PNG uses RGB or RGBA byte order (and big-endian encoding) even when e.g. Windows/x86 typically uses BGR or BGRA and little-endian.

As of a few days ago, decoding WUFFS_BASE__PIXEL_FORMAT__RGB didn't work for all PNGs, because of the "lots of TODOs across the swizzler dispatching code" that you noticed. Those TODOs were fixed since then, by 8954281

It wasn't zero new code, but it was a relatively small amount of new code, since we already supported WUFFS_BASE__PIXEL_FORMAT__BGR and WUFFS_BASE__PIXEL_FORMAT__RGBA_{NONPREMUL,PREMUL}.

despite it's being quite 'natural' (for historical reasons) when it comes to computer vision models, especially neural networks

You probably know more about image decoding in neural networks (e.g OpenCV) than I do. Do you know if they would work with a BGR pixel format or do they only speak RGB?

it seems to me like for wuffs_aux::DecodeImage code it might make sense to just pass through the destination pixel format parameter and let the actual decoder itself fail later while selecting a pixel swizzler. Or am I getting this wrong?

That might make a (small) difference, in theory, but in practice, the decoders just delegate to the pixel swizzler code and it seemed better to fail earlier. It might be worth revisiting that in the future, though. Thanks for the feedback.

from wuffs.

dev0x13 avatar dev0x13 commented on May 24, 2024

Thank you very much for the explanation and for efforts for adding RGB support!

You probably know more about image decoding in neural networks (e.g OpenCV) than I do. Do you know if they would work with a BGR pixel format or do they only speak RGB?

Generally speaking, it depends. Some of the solutions work with RGB and some work with BGR. I'd say they both are equally popular as there is no common convention for channel ordering, so support for both formats is very useful.

That might make a (small) difference, in theory, but in practice, the decoders just delegate to the pixel swizzler code and it seemed better to fail earlier. It might be worth revisiting that in the future, though. Thanks for the feedback.

That makes sense, thank you for explaining this!

from wuffs.

Related Issues (20)

Recommend Projects

  • React photo React

    A declarative, efficient, and flexible JavaScript library for building user interfaces.

  • Vue.js photo Vue.js

    🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.

  • Typescript photo Typescript

    TypeScript is a superset of JavaScript that compiles to clean JavaScript output.

  • TensorFlow photo TensorFlow

    An Open Source Machine Learning Framework for Everyone

  • Django photo Django

    The Web framework for perfectionists with deadlines.

  • D3 photo D3

    Bring data to life with SVG, Canvas and HTML. 📊📈🎉

Recommend Topics

  • javascript

    JavaScript (JS) is a lightweight interpreted programming language with first-class functions.

  • web

    Some thing interesting about web. New door for the world.

  • server

    A server is a program made to process requests and deliver data to clients.

  • Machine learning

    Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.

  • Game

    Some thing interesting about game, make everyone happy.

Recommend Org

  • Facebook photo Facebook

    We are working to build community through open source technology. NB: members must have two-factor auth.

  • Microsoft photo Microsoft

    Open source projects and samples from Microsoft.

  • Google photo Google

    Google ❤️ Open Source for everyone.

  • D3 photo D3

    Data-Driven Documents codes.