An attempt at getting image data back

This commit is contained in:
2024-07-14 00:27:33 +02:00
parent e026bc93f7
commit 6452d2e774
1314 changed files with 218350 additions and 38 deletions

View File

@@ -0,0 +1,3 @@
# SPDX-License-Identifier: GPL-2.0-or-later
#
# Copyright (C) 2022, Paul Elder <paul.elder@ideasonboard.com>

View File

@@ -0,0 +1,6 @@
# SPDX-License-Identifier: GPL-2.0-or-later
#
# Copyright (C) 2024, Paul Elder <paul.elder@ideasonboard.com>
from libtuning.modules.agc.agc import AGC
from libtuning.modules.agc.rkisp1 import AGCRkISP1

View File

@@ -0,0 +1,21 @@
# SPDX-License-Identifier: BSD-2-Clause
#
# Copyright (C) 2019, Raspberry Pi Ltd
# Copyright (C) 2024, Paul Elder <paul.elder@ideasonboard.com>
from ..module import Module
import libtuning as lt
class AGC(Module):
type = 'agc'
hr_name = 'AGC (Base)'
out_name = 'GenericAGC'
# \todo Add sector shapes and stuff just like lsc
def __init__(self, *,
debug: list):
super().__init__()
self.debug = debug

View File

@@ -0,0 +1,79 @@
# SPDX-License-Identifier: BSD-2-Clause
#
# Copyright (C) 2019, Raspberry Pi Ltd
# Copyright (C) 2024, Paul Elder <paul.elder@ideasonboard.com>
#
# rkisp1.py - AGC module for tuning rkisp1
from .agc import AGC
import libtuning as lt
class AGCRkISP1(AGC):
hr_name = 'AGC (RkISP1)'
out_name = 'Agc'
def __init__(self, **kwargs):
super().__init__(**kwargs)
# We don't actually need anything from the config file
def validate_config(self, config: dict) -> bool:
return True
def _generate_metering_modes(self) -> dict:
centre_weighted = [
0, 0, 0, 0, 0,
0, 6, 8, 6, 0,
0, 8, 16, 8, 0,
0, 6, 8, 6, 0,
0, 0, 0, 0, 0
]
spot = [
0, 0, 0, 0, 0,
0, 2, 4, 2, 0,
0, 4, 16, 4, 0,
0, 2, 4, 2, 0,
0, 0, 0, 0, 0
]
matrix = [1 for i in range(0, 25)]
return {
'MeteringCentreWeighted': centre_weighted,
'MeteringSpot': spot,
'MeteringMatrix': matrix
}
def _generate_exposure_modes(self) -> dict:
normal = {'shutter': [100, 10000, 30000, 60000, 120000],
'gain': [2.0, 4.0, 6.0, 6.0, 6.0]}
short = {'shutter': [100, 5000, 10000, 20000, 120000],
'gain': [2.0, 4.0, 6.0, 6.0, 6.0]}
return {'ExposureNormal': normal, 'ExposureShort': short}
def _generate_constraint_modes(self) -> dict:
normal = {'lower': {'qLo': 0.98, 'qHi': 1.0, 'yTarget': 0.5}}
highlight = {
'lower': {'qLo': 0.98, 'qHi': 1.0, 'yTarget': 0.5},
'upper': {'qLo': 0.98, 'qHi': 1.0, 'yTarget': 0.8}
}
return {'ConstraintNormal': normal, 'ConstraintHighlight': highlight}
def _generate_y_target(self) -> list:
return 0.5
def process(self, config: dict, images: list, outputs: dict) -> dict:
output = {}
output['AeMeteringMode'] = self._generate_metering_modes()
output['AeExposureMode'] = self._generate_exposure_modes()
output['AeConstraintMode'] = self._generate_constraint_modes()
output['relativeLuminanceTarget'] = self._generate_y_target()
# \todo Debug functionality
return output

View File

@@ -0,0 +1,6 @@
# SPDX-License-Identifier: GPL-2.0-or-later
#
# Copyright (C) 2024, Paul Elder <paul.elder@ideasonboard.com>
from libtuning.modules.ccm.ccm import CCM
from libtuning.modules.ccm.rkisp1 import CCMRkISP1

View File

@@ -0,0 +1,41 @@
# SPDX-License-Identifier: GPL-2.0-or-later
#
# Copyright (C) 2024, Paul Elder <paul.elder@ideasonboard.com>
# Copyright (C) 2024, Ideas on Board
#
# Base Ccm tuning module
from ..module import Module
from libtuning.ctt_ccm import ccm
import logging
logger = logging.getLogger(__name__)
class CCM(Module):
type = 'ccm'
hr_name = 'CCM (Base)'
out_name = 'GenericCCM'
def __init__(self, debug: list):
super().__init__()
self.debug = debug
def do_calibration(self, images):
logger.info('Starting CCM calibration')
imgs = [img for img in images if img.macbeth is not None]
# todo: Take LSC calibration results into account.
cal_cr_list = None
cal_cb_list = None
try:
ccms = ccm(imgs, cal_cr_list, cal_cb_list)
except ArithmeticError:
logger.error('CCM calibration failed')
return None
return ccms

View File

@@ -0,0 +1,28 @@
# SPDX-License-Identifier: GPL-2.0-or-later
#
# Copyright (C) 2024, Paul Elder <paul.elder@ideasonboard.com>
# Copyright (C) 2024, Ideas on Board
#
# Ccm module for tuning rkisp1
from .ccm import CCM
class CCMRkISP1(CCM):
hr_name = 'Crosstalk Correction (RkISP1)'
out_name = 'Ccm'
def __init__(self, **kwargs):
super().__init__(**kwargs)
# We don't need anything from the config file.
def validate_config(self, config: dict) -> bool:
return True
def process(self, config: dict, images: list, outputs: dict) -> dict:
output = {}
ccms = self.do_calibration(images)
output['ccms'] = ccms
return output

View File

@@ -0,0 +1,7 @@
# SPDX-License-Identifier: GPL-2.0-or-later
#
# Copyright (C) 2022, Paul Elder <paul.elder@ideasonboard.com>
from libtuning.modules.lsc.lsc import LSC
from libtuning.modules.lsc.raspberrypi import ALSCRaspberryPi
from libtuning.modules.lsc.rkisp1 import LSCRkISP1

View File

@@ -0,0 +1,75 @@
# SPDX-License-Identifier: BSD-2-Clause
#
# Copyright (C) 2019, Raspberry Pi Ltd
# Copyright (C) 2022, Paul Elder <paul.elder@ideasonboard.com>
from ..module import Module
import libtuning as lt
import libtuning.utils as utils
import numpy as np
class LSC(Module):
type = 'lsc'
hr_name = 'LSC (Base)'
out_name = 'GenericLSC'
def __init__(self, *,
debug: list,
sector_shape: tuple,
sector_x_gradient: lt.Gradient,
sector_y_gradient: lt.Gradient,
sector_average_function: lt.Average,
smoothing_function: lt.Smoothing):
super().__init__()
self.debug = debug
self.sector_shape = sector_shape
self.sector_x_gradient = sector_x_gradient
self.sector_y_gradient = sector_y_gradient
self.sector_average_function = sector_average_function
self.smoothing_function = smoothing_function
def _enumerate_lsc_images(self, images):
for image in images:
if image.lsc_only:
yield image
def _get_grid(self, channel, img_w, img_h):
# List of number of pixels in each sector
sectors_x = self.sector_x_gradient.distribute(img_w / 2, self.sector_shape[0])
sectors_y = self.sector_y_gradient.distribute(img_h / 2, self.sector_shape[1])
grid = []
r = 0
for y in sectors_y:
c = 0
for x in sectors_x:
grid.append(self.sector_average_function.average(channel[r:r + y, c:c + x]))
c += x
r += y
return np.array(grid)
def _lsc_single_channel(self, channel: np.array,
image: lt.Image, green_grid: np.array = None):
grid = self._get_grid(channel, image.w, image.h)
# Clamp the values to a small positive, so that the following 1/grid
# doesn't produce negative results.
grid = np.maximum(grid - image.blacklevel_16, 0.1)
if green_grid is None:
table = np.reshape(1 / grid, self.sector_shape[::-1])
else:
table = np.reshape(green_grid / grid, self.sector_shape[::-1])
table = self.smoothing_function.smoothing(table)
if green_grid is None:
table = table / np.min(table)
return table, grid

View File

@@ -0,0 +1,248 @@
# SPDX-License-Identifier: BSD-2-Clause
#
# Copyright (C) 2019, Raspberry Pi Ltd
# Copyright (C) 2022, Paul Elder <paul.elder@ideasonboard.com>
#
# ALSC module for tuning Raspberry Pi
from .lsc import LSC
import libtuning as lt
import libtuning.utils as utils
from numbers import Number
import numpy as np
import logging
logger = logging.getLogger(__name__)
class ALSCRaspberryPi(LSC):
# Override the type name so that the parser can match the entry in the
# config file.
type = 'alsc'
hr_name = 'ALSC (Raspberry Pi)'
out_name = 'rpi.alsc'
compatible = ['raspberrypi']
def __init__(self, *,
do_color: lt.Param,
luminance_strength: lt.Param,
**kwargs):
super().__init__(**kwargs)
self.do_color = do_color
self.luminance_strength = luminance_strength
self.output_range = (0, 3.999)
def validate_config(self, config: dict) -> bool:
if self not in config:
logger.error(f'{self.type} not in config')
return False
valid = True
conf = config[self]
lum_key = self.luminance_strength.name
color_key = self.do_color.name
if lum_key not in conf and self.luminance_strength.required:
logger.error(f'{lum_key} is not in config')
valid = False
if lum_key in conf and (conf[lum_key] < 0 or conf[lum_key] > 1):
logger.warning(f'{lum_key} is not in range [0, 1]; defaulting to 0.5')
if color_key not in conf and self.do_color.required:
logger.error(f'{color_key} is not in config')
valid = False
return valid
# @return Image color temperature, flattened array of red calibration table
# (containing {sector size} elements), flattened array of blue
# calibration table, flattened array of green calibration
# table
def _do_single_alsc(self, image: lt.Image, do_alsc_colour: bool):
average_green = np.mean((image.channels[lt.Color.GR:lt.Color.GB + 1]), axis=0)
cg, g = self._lsc_single_channel(average_green, image)
if not do_alsc_colour:
return image.color, None, None, cg.flatten()
cr, _ = self._lsc_single_channel(image.channels[lt.Color.R], image, g)
cb, _ = self._lsc_single_channel(image.channels[lt.Color.B], image, g)
# \todo implement debug
return image.color, cr.flatten(), cb.flatten(), cg.flatten()
# @return Red shading table, Blue shading table, Green shading table,
# number of images processed
def _do_all_alsc(self, images: list, do_alsc_colour: bool, general_conf: dict) -> (list, list, list, Number, int):
# List of colour temperatures
list_col = []
# Associated calibration tables
list_cr = []
list_cb = []
list_cg = []
count = 0
for image in self._enumerate_lsc_images(images):
col, cr, cb, cg = self._do_single_alsc(image, do_alsc_colour)
list_col.append(col)
list_cr.append(cr)
list_cb.append(cb)
list_cg.append(cg)
count += 1
# Convert to numpy array for data manipulation
list_col = np.array(list_col)
list_cr = np.array(list_cr)
list_cb = np.array(list_cb)
list_cg = np.array(list_cg)
cal_cr_list = []
cal_cb_list = []
# Note: Calculation of average corners and center of the shading tables
# has been removed (which ctt had, as it was unused)
# Average all values for luminance shading and return one table for all temperatures
lum_lut = list(np.round(np.mean(list_cg, axis=0), 3))
if not do_alsc_colour:
return None, None, lum_lut, count
for ct in sorted(set(list_col)):
# Average tables for the same colour temperature
indices = np.where(list_col == ct)
ct = int(ct)
t_r = np.round(np.mean(list_cr[indices], axis=0), 3)
t_b = np.round(np.mean(list_cb[indices], axis=0), 3)
cr_dict = {
'ct': ct,
'table': list(t_r)
}
cb_dict = {
'ct': ct,
'table': list(t_b)
}
cal_cr_list.append(cr_dict)
cal_cb_list.append(cb_dict)
return cal_cr_list, cal_cb_list, lum_lut, count
# @brief Calculate sigma from two adjacent gain tables
def _calcSigma(self, g1, g2):
g1 = np.reshape(g1, self.sector_shape[::-1])
g2 = np.reshape(g2, self.sector_shape[::-1])
# Apply gains to gain table
gg = g1 / g2
if np.mean(gg) < 1:
gg = 1 / gg
# For each internal patch, compute average difference between it and
# its 4 neighbours, then append to list
diffs = []
for i in range(self.sector_shape[1] - 2):
for j in range(self.sector_shape[0] - 2):
# Indexing is incremented by 1 since all patches on borders are
# not counted
diff = np.abs(gg[i + 1][j + 1] - gg[i][j + 1])
diff += np.abs(gg[i + 1][j + 1] - gg[i + 2][j + 1])
diff += np.abs(gg[i + 1][j + 1] - gg[i + 1][j])
diff += np.abs(gg[i + 1][j + 1] - gg[i + 1][j + 2])
diffs.append(diff / 4)
mean_diff = np.mean(diffs)
return np.round(mean_diff, 5)
# @brief Obtains sigmas for red and blue, effectively a measure of the
# 'error'
def _get_sigma(self, cal_cr_list, cal_cb_list):
# Provided colour alsc tables were generated for two different colour
# temperatures sigma is calculated by comparing two calibration temperatures
# adjacent in colour space
color_temps = [cal['ct'] for cal in cal_cr_list]
# Calculate sigmas for each adjacent color_temps and return worst one
sigma_rs = []
sigma_bs = []
for i in range(len(color_temps) - 1):
sigma_rs.append(self._calcSigma(cal_cr_list[i]['table'], cal_cr_list[i + 1]['table']))
sigma_bs.append(self._calcSigma(cal_cb_list[i]['table'], cal_cb_list[i + 1]['table']))
# Return maximum sigmas, not necessarily from the same colour
# temperature interval
sigma_r = max(sigma_rs) if sigma_rs else 0.005
sigma_b = max(sigma_bs) if sigma_bs else 0.005
return sigma_r, sigma_b
def process(self, config: dict, images: list, outputs: dict) -> dict:
output = {
'omega': 1.3,
'n_iter': 100,
'luminance_strength': 0.7
}
conf = config[self]
general_conf = config['general']
do_alsc_colour = self.do_color.get_value(conf)
# \todo I have no idea where this input parameter is used
luminance_strength = self.luminance_strength.get_value(conf)
if luminance_strength < 0 or luminance_strength > 1:
luminance_strength = 0.5
output['luminance_strength'] = luminance_strength
# \todo Validate images from greyscale camera and force grescale mode
# \todo Debug functionality
alsc_out = self._do_all_alsc(images, do_alsc_colour, general_conf)
# \todo Handle the second green lut
cal_cr_list, cal_cb_list, luminance_lut, count = alsc_out
if not do_alsc_colour:
output['luminance_lut'] = luminance_lut
output['n_iter'] = 0
return output
output['calibrations_Cr'] = cal_cr_list
output['calibrations_Cb'] = cal_cb_list
output['luminance_lut'] = luminance_lut
# The sigmas determine the strength of the adaptive algorithm, that
# cleans up any lens shading that has slipped through the alsc. These
# are determined by measuring a 'worst-case' difference between two
# alsc tables that are adjacent in colour space. If, however, only one
# colour temperature has been provided, then this difference can not be
# computed as only one table is available.
# To determine the sigmas you would have to estimate the error of an
# alsc table with only the image it was taken on as a check. To avoid
# circularity, dfault exaggerated sigmas are used, which can result in
# too much alsc and is therefore not advised.
# In general, just take another alsc picture at another colour
# temperature!
if count == 1:
output['sigma'] = 0.005
output['sigma_Cb'] = 0.005
logger.warning('Only one alsc calibration found; standard sigmas used for adaptive algorithm.')
return output
# Obtain worst-case scenario residual sigmas
sigma_r, sigma_b = self._get_sigma(cal_cr_list, cal_cb_list)
output['sigma'] = np.round(sigma_r, 5)
output['sigma_Cb'] = np.round(sigma_b, 5)
return output

View File

@@ -0,0 +1,116 @@
# SPDX-License-Identifier: BSD-2-Clause
#
# Copyright (C) 2019, Raspberry Pi Ltd
# Copyright (C) 2022, Paul Elder <paul.elder@ideasonboard.com>
#
# LSC module for tuning rkisp1
from .lsc import LSC
import libtuning as lt
import libtuning.utils as utils
from numbers import Number
import numpy as np
class LSCRkISP1(LSC):
hr_name = 'LSC (RkISP1)'
out_name = 'LensShadingCorrection'
# \todo Not sure if this is useful. Probably will remove later.
compatible = ['rkisp1']
def __init__(self, *args, **kwargs):
super().__init__(**kwargs)
# We don't actually need anything from the config file
def validate_config(self, config: dict) -> bool:
return True
# @return Image color temperature, flattened array of red calibration table
# (containing {sector size} elements), flattened array of blue
# calibration table, flattened array of (red's) green calibration
# table, flattened array of (blue's) green calibration table
def _do_single_lsc(self, image: lt.Image):
# Perform LSC on each colour channel independently. A future enhancement
# worth investigating would be splitting the luminance and chrominance
# LSC as done by Raspberry Pi.
cgr, _ = self._lsc_single_channel(image.channels[lt.Color.GR], image)
cgb, _ = self._lsc_single_channel(image.channels[lt.Color.GB], image)
cr, _ = self._lsc_single_channel(image.channels[lt.Color.R], image)
cb, _ = self._lsc_single_channel(image.channels[lt.Color.B], image)
return image.color, cr.flatten(), cb.flatten(), cgr.flatten(), cgb.flatten()
# @return List of dictionaries of color temperature, red table, red's green
# table, blue's green table, and blue table
def _do_all_lsc(self, images: list) -> list:
output_list = []
output_map_func = lt.gradient.Linear().map
# List of colour temperatures
list_col = []
# Associated calibration tables
list_cr = []
list_cb = []
list_cgr = []
list_cgb = []
for image in self._enumerate_lsc_images(images):
col, cr, cb, cgr, cgb = self._do_single_lsc(image)
list_col.append(col)
list_cr.append(cr)
list_cb.append(cb)
list_cgr.append(cgr)
list_cgb.append(cgb)
# Convert to numpy array for data manipulation
list_col = np.array(list_col)
list_cr = np.array(list_cr)
list_cb = np.array(list_cb)
list_cgr = np.array(list_cgr)
list_cgb = np.array(list_cgb)
for color_temperature in sorted(set(list_col)):
# Average tables for the same colour temperature
indices = np.where(list_col == color_temperature)
color_temperature = int(color_temperature)
tables = []
for lis in [list_cr, list_cgr, list_cgb, list_cb]:
table = np.mean(lis[indices], axis=0)
table = output_map_func((1, 4), (1024, 4096), table)
table = np.clip(table, 1024, 4095)
table = np.round(table).astype('int32').tolist()
tables.append(table)
entry = {
'ct': color_temperature,
'r': tables[0],
'gr': tables[1],
'gb': tables[2],
'b': tables[3],
}
output_list.append(entry)
return output_list
def process(self, config: dict, images: list, outputs: dict) -> dict:
output = {}
# \todo This should actually come from self.sector_{x,y}_gradient
size_gradient = lt.gradient.Linear(lt.Remainder.Float)
output['x-size'] = size_gradient.distribute(0.5, 8)
output['y-size'] = size_gradient.distribute(0.5, 8)
output['sets'] = self._do_all_lsc(images)
if len(output['sets']) == 0:
return None
# \todo Validate images from greyscale camera and force grescale mode
# \todo Debug functionality
return output

View File

@@ -0,0 +1,32 @@
# SPDX-License-Identifier: GPL-2.0-or-later
#
# Copyright (C) 2022, Paul Elder <paul.elder@ideasonboard.com>
#
# Base class for algorithm-specific tuning modules
# @var type Type of the module. Defined in the base module.
# @var out_name The key that will be used for the algorithm in the algorithms
# dictionary in the tuning output file
# @var hr_name Human-readable module name, mostly for debugging
class Module(object):
type = 'base'
hr_name = 'Base Module'
out_name = 'GenericAlgorithm'
def __init__(self):
pass
def validate_config(self, config: dict) -> bool:
raise NotImplementedError
# @brief Do the module's processing
# @param config Full configuration from the input configuration file
# @param images List of images to process
# @param outputs The outputs of all modules that were executed before this
# module. Note that this is an input parameter, and the
# output of this module should be returned directly
# @return Result of the module's processing. It may be empty. None
# indicates failure and that the result should not be used.
def process(self, config: dict, images: list, outputs: dict) -> dict:
raise NotImplementedError

View File

@@ -0,0 +1,24 @@
# SPDX-License-Identifier: GPL-2.0-or-later
#
# Copyright (C) 2024, Ideas on Board
#
# Module implementation for static data
from .module import Module
# This module can be used in cases where the tuning file should contain
# static data.
class StaticModule(Module):
def __init__(self, out_name: str, output: dict = {}):
super().__init__()
self.out_name = out_name
self.hr_name = f'Static {out_name}'
self.type = f'static_{out_name}'
self.output = output
def validate_config(self, config: dict) -> bool:
return True
def process(self, config: dict, images: list, outputs: dict) -> dict:
return self.output