From 7b7fdadd260f958093b3fafebeae2fd0fd06c131 Mon Sep 17 00:00:00 2001 From: Maciej Dudek Date: Thu, 13 Oct 2022 12:12:31 +0200 Subject: [PATCH 1/9] liblitedram: Move pattern testing to its own function Signed-off-by: Maciej Dudek --- litex/soc/software/liblitedram/sdram.c | 27 +++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/litex/soc/software/liblitedram/sdram.c b/litex/soc/software/liblitedram/sdram.c index a1e0bdde3..e80c0a4d6 100644 --- a/litex/soc/software/liblitedram/sdram.c +++ b/litex/soc/software/liblitedram/sdram.c @@ -370,6 +370,19 @@ static unsigned int sdram_write_read_check_test_pattern(int module, unsigned int return errors; } +static int _seed_array[] = {42, 84, 36, 72, 24, 48}; +static int _seed_array_length = sizeof(_seed_array) / sizeof(_seed_array[0]); + + +static int run_test_pattern(int module) +{ + int errors = 0; + for (int i = 0; i < _seed_array_length; i++) { + errors += sdram_write_read_check_test_pattern(module, _seed_array[i]); + } + return errors; +} + static void sdram_leveling_center_module( int module, int show_short, int show_long, delay_callback rst_delay, delay_callback inc_delay) { @@ -387,8 +400,7 @@ static void sdram_leveling_center_module( delay = 0; rst_delay(module); while(1) { - errors = sdram_write_read_check_test_pattern(module, 42); - errors += sdram_write_read_check_test_pattern(module, 84); + errors = run_test_pattern(module); working = errors == 0; show = show_long; #if SDRAM_PHY_DELAYS > 32 @@ -410,8 +422,7 @@ static void sdram_leveling_center_module( cur_delay_min = delay_min; /* Find largest working delay range */ while(1) { - errors = sdram_write_read_check_test_pattern(module, 42); - errors += sdram_write_read_check_test_pattern(module, 84); + errors = run_test_pattern(module); working = errors == 0; show = show_long; #if SDRAM_PHY_DELAYS > 32 @@ -467,8 +478,7 @@ static void sdram_leveling_center_module( } /* Check */ - errors = sdram_write_read_check_test_pattern(module, 42); - errors += sdram_write_read_check_test_pattern(module, 84); + errors = run_test_pattern(module); if (errors == 0) break; retries--; @@ -949,7 +959,7 @@ static void sdram_read_leveling_inc_bitslip(char m) static unsigned int sdram_read_leveling_scan_module(int module, int bitslip, int show) { - const unsigned int max_errors = 2*READ_CHECK_TEST_PATTERN_MAX_ERRORS; + const unsigned int max_errors = _seed_array_length * READ_CHECK_TEST_PATTERN_MAX_ERRORS; int i; unsigned int score; unsigned int errors; @@ -965,8 +975,7 @@ static unsigned int sdram_read_leveling_scan_module(int module, int bitslip, int #if SDRAM_PHY_DELAYS > 32 _show = (i%16 == 0) & show; #endif - errors = sdram_write_read_check_test_pattern(module, 42); - errors += sdram_write_read_check_test_pattern(module, 84); + errors = run_test_pattern(module); working = errors == 0; /* When any scan is working then the final score will always be higher then if no scan was working */ score += (working * max_errors*SDRAM_PHY_DELAYS) + (max_errors - errors); From 48f547885a2099f1edb65cb9fb3a275e6641db41 Mon Sep 17 00:00:00 2001 From: Maciej Dudek Date: Fri, 15 Jul 2022 18:09:32 +0200 Subject: [PATCH 2/9] liblitedram: Correctly support single x4 IC Signed-off-by: Maciej Dudek --- litex/soc/software/liblitedram/sdram.c | 54 ++++++++++++++++++-------- 1 file changed, 38 insertions(+), 16 deletions(-) diff --git a/litex/soc/software/liblitedram/sdram.c b/litex/soc/software/liblitedram/sdram.c index e80c0a4d6..86cdfa303 100644 --- a/litex/soc/software/liblitedram/sdram.c +++ b/litex/soc/software/liblitedram/sdram.c @@ -298,6 +298,7 @@ static void print_scan_errors(unsigned int errors) { } #define READ_CHECK_TEST_PATTERN_MAX_ERRORS (8*SDRAM_PHY_PHASES*DFII_PIX_DATA_BYTES/SDRAM_PHY_MODULES) +#define MODULE_BITMASK ((1<> ibo) & mask) ^ + ((tst[pebo] >> ibo) & mask)); + if (SDRAM_PHY_DQ_DQS_RATIO == 16) + errors += popcount(((prs[p][pebo+1] >> ibo) & mask) ^ + ((tst[pebo+1] >> ibo) & mask)); + + +#if SDRAM_PHY_XDR == 2 + if (DFII_PIX_DATA_BYTES == 1) // Special case for x4 single IC + ibo = 0x4; + errors += popcount(((prs[p][nebo] >> ibo) & mask) ^ + ((tst[nebo] >> ibo) & mask)); + if (SDRAM_PHY_DQ_DQS_RATIO == 16) + errors += popcount(((prs[p][nebo+1] >> ibo) & mask) ^ + ((tst[nebo+1] >> ibo) & mask)); +#endif //SDRAM_PHY_XDR == 2 } #if defined(SDRAM_PHY_ECP5DDRPHY) || defined(SDRAM_PHY_GW2DDRPHY) From cd89b62ff65a3eaba78a05b6b15d51835adc7bac Mon Sep 17 00:00:00 2001 From: Maciej Dudek Date: Wed, 25 Jan 2023 11:35:20 +0100 Subject: [PATCH 3/9] liblitedram: Fix LFSR LFSR only returns 1 random bit per lfsr() call, so we should concatenate 8 consecutive calls to obtain 1 random byte Signed-off-by: Maciej Dudek --- litex/soc/software/liblitedram/sdram.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/litex/soc/software/liblitedram/sdram.c b/litex/soc/software/liblitedram/sdram.c index 86cdfa303..aea05759e 100644 --- a/litex/soc/software/liblitedram/sdram.c +++ b/litex/soc/software/liblitedram/sdram.c @@ -301,9 +301,10 @@ static void print_scan_errors(unsigned int errors) { #define MODULE_BITMASK ((1< Date: Wed, 25 Jan 2023 11:38:13 +0100 Subject: [PATCH 4/9] liblitedram: discard singular 1s that are surrounded by 0s They are most likely glitches and won't be stable. Signed-off-by: Maciej Dudek --- litex/soc/software/liblitedram/sdram.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/litex/soc/software/liblitedram/sdram.c b/litex/soc/software/liblitedram/sdram.c index aea05759e..358df8535 100644 --- a/litex/soc/software/liblitedram/sdram.c +++ b/litex/soc/software/liblitedram/sdram.c @@ -415,7 +415,7 @@ static void sdram_leveling_center_module( { int i; int show; - int working; + int working, last_working; unsigned int errors; int delay, delay_mid, delay_range; int delay_min = -1, delay_max = -1, cur_delay_min = -1; @@ -425,9 +425,11 @@ static void sdram_leveling_center_module( /* Find smallest working delay */ delay = 0; + working = 0; rst_delay(module); while(1) { errors = run_test_pattern(module); + last_working = working; working = errors == 0; show = show_long; #if SDRAM_PHY_DELAYS > 32 @@ -435,8 +437,8 @@ static void sdram_leveling_center_module( #endif if (show) print_scan_errors(errors); - if(working && delay_min < 0) { - delay_min = delay; + if(working && last_working && delay_min < 0) { + delay_min = delay - 1; // delay on edges can be spotty break; } delay++; From 8e848f6a133431284accc35ecae18aa96d28df86 Mon Sep 17 00:00:00 2001 From: Maciej Dudek Date: Tue, 24 Jan 2023 19:44:32 +0100 Subject: [PATCH 5/9] liblitedram: #define MODULO It allows to replace repeating #ifdef SDRAM_PHY_DELAYS > 32 Signed-off-by: Maciej Dudek --- litex/soc/software/liblitedram/sdram.c | 27 +++++++++++--------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/litex/soc/software/liblitedram/sdram.c b/litex/soc/software/liblitedram/sdram.c index 358df8535..688706996 100644 --- a/litex/soc/software/liblitedram/sdram.c +++ b/litex/soc/software/liblitedram/sdram.c @@ -34,6 +34,12 @@ #ifdef CSR_SDRAM_BASE +#if SDRAM_PHY_DELAYS > 32 +#define MODULO (SDRAM_PHY_DELAYS/32) +#else +#define MODULO (1) +#endif + /*-----------------------------------------------------------------------*/ /* Helpers */ /*-----------------------------------------------------------------------*/ @@ -431,10 +437,7 @@ static void sdram_leveling_center_module( errors = run_test_pattern(module); last_working = working; working = errors == 0; - show = show_long; -#if SDRAM_PHY_DELAYS > 32 - show = show && (delay%16 == 0); -#endif + show = show_long && (delay%MODULO == 0); if (show) print_scan_errors(errors); if(working && last_working && delay_min < 0) { @@ -453,10 +456,7 @@ static void sdram_leveling_center_module( while(1) { errors = run_test_pattern(module); working = errors == 0; - show = show_long; -#if SDRAM_PHY_DELAYS > 32 - show = show && (delay%16 == 0); -#endif + show = show_long && (delay%MODULO == 0); if (show) print_scan_errors(errors); @@ -674,10 +674,8 @@ static int sdram_write_leveling_scan(int *delays, int loops, int show) for(j=0;j 32 - show_iter = (j%16 == 0) && show; -#endif + int show_iter = (j%MODULO == 0) && show; + for (k=0; k 32 - _show = (i%16 == 0) & show; -#endif + int _show = (i%MODULO == 0) & show; errors = run_test_pattern(module); working = errors == 0; /* When any scan is working then the final score will always be higher then if no scan was working */ From 25b502dec8b6205a96fc75d52689fd2ba9fd48bb Mon Sep 17 00:00:00 2001 From: Maciej Dudek Date: Tue, 24 Jan 2023 19:51:56 +0100 Subject: [PATCH 6/9] liblitedram: define SDRAM_WLC_DEBUG SDRAM_WLC_DEBUG is 0 or 1 based on SDRAM_WRITE_LATENCY_CALIBRATION_DEBUG. This reduces number of #ifdefs Signed-off-by: Maciej Dudek --- litex/soc/software/liblitedram/sdram.c | 29 +++++++++++++++----------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/litex/soc/software/liblitedram/sdram.c b/litex/soc/software/liblitedram/sdram.c index 688706996..6c14c6143 100644 --- a/litex/soc/software/liblitedram/sdram.c +++ b/litex/soc/software/liblitedram/sdram.c @@ -34,6 +34,12 @@ #ifdef CSR_SDRAM_BASE +#ifdef SDRAM_WRITE_LATENCY_CALIBRATION_DEBUG +#define SDRAM_WLC_DEBUG 1 +#else +#define SDRAM_WLC_DEBUG 0 +#endif + #if SDRAM_PHY_DELAYS > 32 #define MODULO (SDRAM_PHY_DELAYS/32) #else @@ -1085,9 +1091,9 @@ static void sdram_write_latency_calibration(void) { best_score = 0; best_bitslip = -1; for(bitslip=0; bitslip score ? subscore : score; /* Increment bitslip */ sdram_read_leveling_inc_bitslip(module); @@ -1130,9 +1135,9 @@ static void sdram_write_latency_calibration(void) { printf("m%d:- ", module); else printf("m%d:%d ", module, bitslip); -#ifdef SDRAM_WRITE_LATENCY_CALIBRATION_DEBUG - printf("\n"); -#endif + + if (SDRAM_WLC_DEBUG) + printf("\n"); /* Select best write window */ ddrphy_dly_sel_write(1 << module); From 58fda1a385ae5f88e8bdb7927155042703d363d1 Mon Sep 17 00:00:00 2001 From: Maciej Dudek Date: Wed, 25 Jan 2023 14:09:28 +0100 Subject: [PATCH 7/9] liblitedram: Move PHY accessors to a separate file They are independent from the training flow. Signed-off-by: Maciej Dudek --- litex/soc/software/bios/cmds/cmd_litedram.c | 6 +- litex/soc/software/liblitedram/Makefile | 2 +- litex/soc/software/liblitedram/accessors.c | 187 ++++++++++++++ litex/soc/software/liblitedram/accessors.h | 77 ++++++ litex/soc/software/liblitedram/sdram.c | 262 ++++---------------- litex/soc/software/liblitedram/sdram.h | 4 - 6 files changed, 316 insertions(+), 222 deletions(-) create mode 100644 litex/soc/software/liblitedram/accessors.c create mode 100644 litex/soc/software/liblitedram/accessors.h diff --git a/litex/soc/software/bios/cmds/cmd_litedram.c b/litex/soc/software/bios/cmds/cmd_litedram.c index 4420bacea..f086f7cc1 100644 --- a/litex/soc/software/bios/cmds/cmd_litedram.c +++ b/litex/soc/software/bios/cmds/cmd_litedram.c @@ -14,6 +14,7 @@ #include #include #include +#include #include "../command.h" #include "../helpers.h" @@ -228,7 +229,7 @@ define_command(sdram_force_cmd_delay, sdram_force_cmd_delay_handler, "Force writ #endif -#ifdef CSR_DDRPHY_WDLY_DQ_RST_ADDR +#ifdef SDRAM_PHY_WRITE_LEVELING_CAPABLE /** * Command "sdram_rst_dat_delay" @@ -290,6 +291,9 @@ static void sdram_force_dat_delay_handler(int nb_params, char **params) define_command(sdram_force_dat_delay, sdram_force_dat_delay_handler, "Force write leveling Dat delay", LITEDRAM_CMDS); #endif +#endif + +#ifdef SDRAM_PHY_BITSLIPS /** * Command "sdram_rst_bitslip" * diff --git a/litex/soc/software/liblitedram/Makefile b/litex/soc/software/liblitedram/Makefile index 51c3e8952..664685be3 100644 --- a/litex/soc/software/liblitedram/Makefile +++ b/litex/soc/software/liblitedram/Makefile @@ -1,7 +1,7 @@ include ../include/generated/variables.mak include $(SOC_DIRECTORY)/software/common.mak -OBJECTS = sdram.o bist.o sdram_dbg.o sdram_spd.o utils.o +OBJECTS = sdram.o bist.o sdram_dbg.o sdram_spd.o utils.o accessors.o all: liblitedram.a diff --git a/litex/soc/software/liblitedram/accessors.c b/litex/soc/software/liblitedram/accessors.c new file mode 100644 index 000000000..fb0c48b52 --- /dev/null +++ b/litex/soc/software/liblitedram/accessors.c @@ -0,0 +1,187 @@ +#include + +#include + +#if defined(CSR_SDRAM_BASE) && defined(CSR_DDRPHY_BASE) + +#if defined(SDRAM_PHY_READ_LEVELING_CAPABLE) + +int read_dq_delay[SDRAM_PHY_MODULES]; +void read_inc_dq_delay(int module) { + /* Increment delay */ + read_dq_delay[module] = (read_dq_delay[module] + 1) & (SDRAM_PHY_DELAYS - 1); + ddrphy_rdly_dq_inc_write(1); +} + +void read_rst_dq_delay(int module) { + /* Reset delay */ + read_dq_delay[module] = 0; + ddrphy_rdly_dq_rst_write(1); +} + +#endif // defined(SDRAM_PHY_READ_LEVELING_CAPABLE) + +#if defined(SDRAM_PHY_WRITE_LEVELING_CAPABLE) + +int sdram_clock_delay; +void sdram_inc_clock_delay(void) { + sdram_clock_delay = (sdram_clock_delay + 1) & (SDRAM_PHY_DELAYS - 1); + ddrphy_cdly_inc_write(1); + cdelay(100); +} + +void sdram_rst_clock_delay(void) { + sdram_clock_delay = 0; + ddrphy_cdly_rst_write(1); + cdelay(100); +} + +int write_dq_delay[SDRAM_PHY_MODULES]; +void write_inc_dq_delay(int module) { + /* Increment DQ delay */ + write_dq_delay[module] = (write_dq_delay[module] + 1) & (SDRAM_PHY_DELAYS - 1); + ddrphy_wdly_dq_inc_write(1); + cdelay(100); +} + +void write_rst_dq_delay(int module) { +#if defined(SDRAM_PHY_USDDRPHY) || defined(SDRAM_PHY_USPDDRPHY) + /* Reset DQ delay */ + int dq_count = ddrphy_wdly_dqs_inc_count_read(); + while (dq_count != SDRAM_PHY_DELAYS) { + ddrphy_wdly_dq_inc_write(1); + cdelay(100); + dq_count++; + } +#else + /* Reset DQ delay */ + ddrphy_wdly_dq_rst_write(1); + cdelay(100); +#endif //defined(SDRAM_PHY_USDDRPHY) || defined(SDRAM_PHY_USPDDRPHY) + write_dq_delay[module] = 0; +} + +void write_inc_dqs_delay(int module) { + /* Increment DQS delay */ + ddrphy_wdly_dqs_inc_write(1); + cdelay(100); +} + +void write_rst_dqs_delay(int module) { +#if defined(SDRAM_PHY_USDDRPHY) || defined(SDRAM_PHY_USPDDRPHY) + /* Reset DQS delay */ + while (ddrphy_wdly_dqs_inc_count_read() != 0) { + ddrphy_wdly_dqs_inc_write(1); + cdelay(100); + } +#else + /* Reset DQS delay */ + ddrphy_wdly_dqs_rst_write(1); + cdelay(100); +#endif //defined(SDRAM_PHY_USDDRPHY) || defined(SDRAM_PHY_USPDDRPHY) +} + +void write_inc_delay(int module) { + /* Increment DQ/DQS delay */ + write_inc_dq_delay(module); + write_inc_dqs_delay(module); +} + +void write_rst_delay(int module) { + write_rst_dq_delay(module); + write_rst_dqs_delay(module); +} + +#endif // defined(SDRAM_PHY_WRITE_LEVELING_CAPABLE) + +#if defined(SDRAM_PHY_WRITE_LATENCY_CALIBRATION_CAPABLE) || defined(SDRAM_BITSLIP_CAPABLE) + +int read_dq_bitslip[SDRAM_PHY_MODULES]; +void read_inc_dq_bitslip(int module) { + /* Increment bitslip */ + read_dq_bitslip[module] = (read_dq_bitslip[module] + 1) & (SDRAM_PHY_BITSLIPS - 1); + ddrphy_rdly_dq_bitslip_write(1); +} + +void read_rst_dq_bitslip(int module) { +/* Reset bitslip */ + read_dq_bitslip[module] = 0; + ddrphy_rdly_dq_bitslip_rst_write(1); +} + +int write_dq_bitslip[SDRAM_PHY_MODULES]; +void write_inc_dq_bitslip(int module) { + /* Increment bitslip */ + write_dq_bitslip[module] = (write_dq_bitslip[module] + 1) & (SDRAM_PHY_BITSLIPS - 1); + ddrphy_wdly_dq_bitslip_write(1); +} + +void write_rst_dq_bitslip(int module) { + /* Increment bitslip */ + write_dq_bitslip[module] = 0; + ddrphy_wdly_dq_bitslip_rst_write(1); +} + +#endif // defined(SDRAM_PHY_WRITE_LATENCY_CALIBRATION_CAPABLE) || defined(SDRAM_BITSLIP_CAPABLE) + +#if defined(SDRAM_PHY_WRITE_LEVELING_CAPABLE) || defined(SDRAM_PHY_WRITE_LATENCY_CALIBRATION_CAPABLE) || \ +defined(SDRAM_PHY_READ_LEVELING_CAPABLE) || defined(SDRAM_BITSLIP_CAPABLE) + +void sdram_select(int module) { + ddrphy_dly_sel_write(1 << module); +} + +void sdram_deselect(int module) { + ddrphy_dly_sel_write(0); + +#if defined(SDRAM_PHY_ECP5DDRPHY) || defined(SDRAM_PHY_GW2DDRPHY) + /* Sync all DQSBUFM's, By toggling all dly_sel (DQSBUFM.PAUSE) lines. */ + ddrphy_dly_sel_write(0xff); + ddrphy_dly_sel_write(0); +#endif //SDRAM_PHY_ECP5DDRPHY +} + +void sdram_leveling_action(int module, action_callback action) { + /* Select module */ + sdram_select(module); + + /* Action */ + action(module); + + /* Un-select module */ + sdram_deselect(module); +} + +#endif // defined(SDRAM_PHY_WRITE_LEVELING_CAPABLE) || defined(SDRAM_PHY_WRITE_LATENCY_CALIBRATION_CAPABLE) || defined(SDRAM_PHY_READ_LEVELING_CAPABLE) ... + +#ifdef SDRAM_PHY_WRITE_LEVELING_CAPABLE + +int _sdram_write_leveling_bitslips[16]; +int _sdram_write_leveling_dat_delays[16]; + +void sdram_write_leveling_rst_dat_delay(int module, int show) { + _sdram_write_leveling_dat_delays[module] = -1; + if (show) + printf("Reseting Dat delay of module %d\n", module); +} + +void sdram_write_leveling_force_dat_delay(int module, int taps, int show) { + _sdram_write_leveling_dat_delays[module] = taps; + if (show) + printf("Forcing Dat delay of module %d to %d taps\n", module, taps); +} + +void sdram_write_leveling_rst_bitslip(int module, int show) { + _sdram_write_leveling_bitslips[module] = -1; + if (show) + printf("Reseting Bitslip of module %d\n", module); +} + +void sdram_write_leveling_force_bitslip(int module, int bitslip, int show) { + _sdram_write_leveling_bitslips[module] = bitslip; + if (show) + printf("Forcing Bitslip of module %d to %d\n", module, bitslip); +} +#endif // SDRAM_PHY_WRITE_LEVELING_CAPABLE + +#endif // defined(CSR_SDRAM_BASE) && defined(CSR_DDRPHY_BASE) diff --git a/litex/soc/software/liblitedram/accessors.h b/litex/soc/software/liblitedram/accessors.h new file mode 100644 index 000000000..7083d2c27 --- /dev/null +++ b/litex/soc/software/liblitedram/accessors.h @@ -0,0 +1,77 @@ +#ifndef __ACCESSORS_H +#define __ACCESSORS_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#if defined(CSR_SDRAM_BASE) && defined(CSR_DDRPHY_BASE) +#include + +typedef void (*action_callback)(int module); + +#if defined(SDRAM_PHY_READ_LEVELING_CAPABLE) + +extern int read_dq_delay[SDRAM_PHY_MODULES]; +void read_inc_dq_delay(int module); +void read_rst_dq_delay(int module); + +#endif // defined(SDRAM_PHY_READ_LEVELING_CAPABLE) + +#if defined(SDRAM_PHY_WRITE_LEVELING_CAPABLE) + +extern int sdram_clock_delay; +void sdram_inc_clock_delay(void); +void sdram_rst_clock_delay(void); + +extern int write_dq_delay[SDRAM_PHY_MODULES]; +void write_inc_dq_delay(int module); +void write_rst_dq_delay(int module); + +void write_inc_dqs_delay(int module); +void write_rst_dqs_delay(int module); + +void write_inc_delay(int module); +void write_rst_delay(int module); + +#endif // defined(SDRAM_PHY_WRITE_LEVELING_CAPABLE) + +#if defined(SDRAM_PHY_WRITE_LATENCY_CALIBRATION_CAPABLE) || defined(SDRAM_BITSLIP_CAPABLE) + +extern int read_dq_bitslip[SDRAM_PHY_MODULES]; +void read_inc_dq_bitslip(int module); +void read_rst_dq_bitslip(int module); + +extern int write_dq_bitslip[SDRAM_PHY_MODULES]; +void write_inc_dq_bitslip(int module); +void write_rst_dq_bitslip(int module); + +#endif // defined(SDRAM_PHY_WRITE_LATENCY_CALIBRATION_CAPABLE) || defined(SDRAM_BITSLIP_CAPABLE) + +#if defined(SDRAM_PHY_WRITE_LEVELING_CAPABLE) || defined(SDRAM_PHY_WRITE_LATENCY_CALIBRATION_CAPABLE) || \ +defined(SDRAM_PHY_READ_LEVELING_CAPABLE) || defined(SDRAM_BITSLIP_CAPABLE) + +void sdram_select(int module); +void sdram_deselect(int module); +void sdram_leveling_action(int module, action_callback action); + +#endif // defined(SDRAM_PHY_WRITE_LEVELING_CAPABLE) || defined(SDRAM_PHY_WRITE_LATENCY_CALIBRATION_CAPABLE) || defined(SDRAM_PHY_READ_LEVELING_CAPABLE) ... + +#ifdef SDRAM_PHY_WRITE_LEVELING_CAPABLE +extern int _sdram_write_leveling_bitslips[16]; +extern int _sdram_write_leveling_dat_delays[16]; + +void sdram_write_leveling_rst_dat_delay(int module, int show); +void sdram_write_leveling_force_dat_delay(int module, int taps, int show); +void sdram_write_leveling_rst_bitslip(int module, int show); +void sdram_write_leveling_force_bitslip(int module, int bitslip, int show); +#endif // SDRAM_PHY_WRITE_LEVELING_CAPABLE + +#endif // defined(CSR_SDRAM_BASE) && defined(CSR_DDRPHY_BASE) + +#ifdef __cplusplus +} +#endif + +#endif // __ACCESSORS_H diff --git a/litex/soc/software/liblitedram/sdram.c b/litex/soc/software/liblitedram/sdram.c index 6c14c6143..1bba4b745 100644 --- a/litex/soc/software/liblitedram/sdram.c +++ b/litex/soc/software/liblitedram/sdram.c @@ -26,6 +26,7 @@ #include #include +#include //#define SDRAM_TEST_DISABLE //#define SDRAM_WRITE_LEVELING_CMD_DELAY_DEBUG @@ -267,8 +268,6 @@ void sdram_mode_register_write(char reg, int value) { /* Leveling Centering (Common for Read/Write Leveling) */ /*-----------------------------------------------------------------------*/ -typedef void (*delay_callback)(int module); - static void sdram_activate_test_row(void) { sdram_dfii_pi0_address_write(0); sdram_dfii_pi0_baddress_write(0); @@ -423,7 +422,7 @@ static int run_test_pattern(int module) } static void sdram_leveling_center_module( - int module, int show_short, int show_long, delay_callback rst_delay, delay_callback inc_delay) + int module, int show_short, int show_long, action_callback rst_delay, action_callback inc_delay) { int i; int show; @@ -438,7 +437,7 @@ static void sdram_leveling_center_module( /* Find smallest working delay */ delay = 0; working = 0; - rst_delay(module); + sdram_leveling_action(module, rst_delay); while(1) { errors = run_test_pattern(module); last_working = working; @@ -453,7 +452,7 @@ static void sdram_leveling_center_module( delay++; if(delay >= SDRAM_PHY_DELAYS) break; - inc_delay(module); + sdram_leveling_action(module, inc_delay); } delay_max = delay_min; @@ -479,7 +478,7 @@ static void sdram_leveling_center_module( delay++; if(delay >= SDRAM_PHY_DELAYS) break; - inc_delay(module); + sdram_leveling_action(module, inc_delay); } if(delay_max < 0) { delay_max = delay; @@ -505,10 +504,10 @@ static void sdram_leveling_center_module( int retries = 8; /* Do N configs/checks and give up if failing */ while (retries > 0) { /* Set delay. */ - rst_delay(module); + sdram_leveling_action(module, rst_delay); cdelay(100); for(i = 0; i < delay_mid; i++) { - inc_delay(module); + sdram_leveling_action(module, inc_delay); cdelay(100); } @@ -526,13 +525,11 @@ static void sdram_leveling_center_module( /*-----------------------------------------------------------------------*/ int _sdram_tck_taps; -int _sdram_write_leveling_bitslips[16]; #ifdef SDRAM_PHY_WRITE_LEVELING_CAPABLE int _sdram_write_leveling_cmd_scan = 1; int _sdram_write_leveling_cmd_delay = 0; -int _sdram_write_leveling_dat_delays[16]; int _sdram_write_leveling_cdly_range_start = -1; int _sdram_write_leveling_cdly_range_end = -1; @@ -580,74 +577,12 @@ void sdram_write_leveling_force_cmd_delay(int taps, int show) { _sdram_write_leveling_cmd_delay = taps; if (show) printf("Forcing Cmd delay to %d taps\n", taps); - ddrphy_cdly_rst_write(1); - cdelay(100); + sdram_rst_clock_delay(); for (i=0; i= 0) { - ddrphy_cdly_rst_write(1); - cdelay(100); + sdram_rst_clock_delay(); for (int i = 0; i < best_cdly; ++i) { - ddrphy_cdly_inc_write(1); - cdelay(100); + sdram_inc_clock_delay(); } } @@ -931,65 +859,6 @@ int sdram_write_leveling(void) /* Read Leveling */ /*-----------------------------------------------------------------------*/ -static void sdram_read_leveling_rst_delay(int module) { - /* Select module */ - ddrphy_dly_sel_write(1 << module); - - /* Reset delay */ - ddrphy_rdly_dq_rst_write(1); - - /* Un-select module */ - ddrphy_dly_sel_write(0); - -#if defined(SDRAM_PHY_ECP5DDRPHY) || defined(SDRAM_PHY_GW2DDRPHY) - /* Sync all DQSBUFM's, By toggling all dly_sel (DQSBUFM.PAUSE) lines. */ - ddrphy_dly_sel_write(0xff); - ddrphy_dly_sel_write(0); -#endif -} - -static void sdram_read_leveling_inc_delay(int module) { - /* Select module */ - ddrphy_dly_sel_write(1 << module); - - /* Increment delay */ - ddrphy_rdly_dq_inc_write(1); - - /* Un-select module */ - ddrphy_dly_sel_write(0); - -#if defined(SDRAM_PHY_ECP5DDRPHY) || defined(SDRAM_PHY_GW2DDRPHY) - /* Sync all DQSBUFM's, By toggling all dly_sel (DQSBUFM.PAUSE) lines. */ - ddrphy_dly_sel_write(0xff); - ddrphy_dly_sel_write(0); -#endif -} - -static void sdram_read_leveling_rst_bitslip(char m) -{ - /* Select module */ - ddrphy_dly_sel_write(1 << m); - - /* Reset delay */ - ddrphy_rdly_dq_bitslip_rst_write(1); - - /* Un-select module */ - ddrphy_dly_sel_write(0); -} - - -static void sdram_read_leveling_inc_bitslip(char m) -{ - /* Select module */ - ddrphy_dly_sel_write(1 << m); - - /* Increment delay */ - ddrphy_rdly_dq_bitslip_write(1); - - /* Un-select module */ - ddrphy_dly_sel_write(0); -} - static unsigned int sdram_read_leveling_scan_module(int module, int bitslip, int show) { const unsigned int max_errors = _seed_array_length * READ_CHECK_TEST_PATTERN_MAX_ERRORS; @@ -1001,7 +870,7 @@ static unsigned int sdram_read_leveling_scan_module(int module, int bitslip, int score = 0; if (show) printf(" m%d, b%02d: |", module, bitslip); - sdram_read_leveling_rst_delay(module); + sdram_leveling_action(module, read_rst_dq_delay); for(i=0;i best_score) { best_bitslip = bitslip; @@ -1055,18 +924,18 @@ void sdram_read_leveling(void) if (bitslip == SDRAM_PHY_BITSLIPS-1) break; /* Increment bitslip */ - sdram_read_leveling_inc_bitslip(module); + sdram_leveling_action(module, read_inc_dq_bitslip); } /* Select best read window */ printf(" best: m%d, b%02d ", module, best_bitslip); - sdram_read_leveling_rst_bitslip(module); + sdram_leveling_action(module, read_rst_dq_bitslip); for (bitslip=0; bitslip score ? subscore : score; /* Increment bitslip */ - sdram_read_leveling_inc_bitslip(module); + sdram_leveling_action(module, read_inc_dq_bitslip); } if (score > best_score) { best_bitslip = bitslip; @@ -1139,16 +1005,11 @@ static void sdram_write_latency_calibration(void) { if (SDRAM_WLC_DEBUG) printf("\n"); - /* Select best write window */ - ddrphy_dly_sel_write(1 << module); - /* Reset bitslip */ - ddrphy_wdly_dq_bitslip_rst_write(1); + sdram_leveling_action(module, write_rst_dq_bitslip); for (i=0; i best_score) { best_bitslip = bitslip; best_score = score; } if (bitslip == SDRAM_PHY_BITSLIPS-1) break; - sdram_read_leveling_inc_bitslip(module); + sdram_leveling_action(module, read_inc_dq_bitslip); } /* Select best read window and re-center it */ - sdram_read_leveling_rst_bitslip(module); + sdram_leveling_action(module, read_rst_dq_bitslip); for (bitslip=0; bitslip Date: Thu, 26 Jan 2023 01:04:54 +0100 Subject: [PATCH 8/9] liblitedram: Codebase cleanup Fix curly braces to be in the same line as the function header. Fix function formating to be consistent in files. Fix guards and add comments at the end of them. Signed-off-by: Maciej Dudek --- litex/soc/software/bios/cmds/cmd_litedram.c | 97 +++++---- litex/soc/software/liblitedram/accessors.c | 14 +- litex/soc/software/liblitedram/accessors.h | 15 +- litex/soc/software/liblitedram/sdram.c | 223 +++++++++----------- 4 files changed, 162 insertions(+), 187 deletions(-) diff --git a/litex/soc/software/bios/cmds/cmd_litedram.c b/litex/soc/software/bios/cmds/cmd_litedram.c index f086f7cc1..f0c9e46f5 100644 --- a/litex/soc/software/bios/cmds/cmd_litedram.c +++ b/litex/soc/software/bios/cmds/cmd_litedram.c @@ -19,46 +19,6 @@ #include "../command.h" #include "../helpers.h" -/** - * Command "sdram_init" - * - * Initialize SDRAM (Init + Calibration) - * - */ -#if defined(CSR_SDRAM_BASE) -define_command(sdram_init, sdram_init, "Initialize SDRAM (Init + Calibration)", LITEDRAM_CMDS); -#endif - -/** - * Command "sdram_cal" - * - * Calibrate SDRAM - * - */ -#if defined(CSR_SDRAM_BASE) && defined(CSR_DDRPHY_BASE) -static void sdram_cal_handler(int nb_params, char **params) -{ - sdram_software_control_on(); - sdram_leveling(); - sdram_software_control_off(); -} -define_command(sdram_cal, sdram_cal_handler, "Calibrate SDRAM", LITEDRAM_CMDS); -#endif - -/** - * Command "sdram_test" - * - * Test SDRAM - * - */ -#if defined(CSR_SDRAM_BASE) -static void sdram_test_handler(int nb_params, char **params) -{ - memtest((unsigned int *)MAIN_RAM_BASE, MAIN_RAM_SIZE/32); -} -define_command(sdram_test, sdram_test_handler, "Test SDRAM", LITEDRAM_CMDS); -#endif - /** * Command "sdram_bist" * @@ -229,6 +189,43 @@ define_command(sdram_force_cmd_delay, sdram_force_cmd_delay_handler, "Force writ #endif +#if defined(CSR_SDRAM_BASE) +/** + * Command "sdram_init" + * + * Initialize SDRAM (Init + Calibration) + * + */ +define_command(sdram_init, sdram_init, "Initialize SDRAM (Init + Calibration)", LITEDRAM_CMDS); + +/** + * Command "sdram_test" + * + * Test SDRAM + * + */ +static void sdram_test_handler(int nb_params, char **params) +{ + memtest((unsigned int *)MAIN_RAM_BASE, MAIN_RAM_SIZE/32); +} +define_command(sdram_test, sdram_test_handler, "Test SDRAM", LITEDRAM_CMDS); + +/** + * Command "sdram_cal" + * + * Calibrate SDRAM + * + */ +#if defined(CSR_DDRPHY_BASE) +static void sdram_cal_handler(int nb_params, char **params) +{ + sdram_software_control_on(); + sdram_leveling(); + sdram_software_control_off(); +} +define_command(sdram_cal, sdram_cal_handler, "Calibrate SDRAM", LITEDRAM_CMDS); +#endif + #ifdef SDRAM_PHY_WRITE_LEVELING_CAPABLE /** @@ -237,7 +234,7 @@ define_command(sdram_force_cmd_delay, sdram_force_cmd_delay_handler, "Force writ * Reset write leveling Dat delay * */ -#if defined(CSR_SDRAM_BASE) && defined(CSR_DDRPHY_BASE) +#if defined(CSR_DDRPHY_BASE) static void sdram_rst_dat_delay_handler(int nb_params, char **params) { char *c; @@ -264,7 +261,7 @@ define_command(sdram_rst_dat_delay, sdram_rst_dat_delay_handler, "Reset write le * Force write leveling Dat delay * */ -#if defined(CSR_SDRAM_BASE) && defined(CSR_DDRPHY_BASE) +#if defined(CSR_DDRPHY_BASE) static void sdram_force_dat_delay_handler(int nb_params, char **params) { char *c; @@ -289,18 +286,18 @@ static void sdram_force_dat_delay_handler(int nb_params, char **params) sdram_software_control_off(); } define_command(sdram_force_dat_delay, sdram_force_dat_delay_handler, "Force write leveling Dat delay", LITEDRAM_CMDS); -#endif +#endif /* defined(CSR_SDRAM_BASE) && defined(CSR_DDRPHY_BASE) */ -#endif +#endif /* SDRAM_PHY_WRITE_LEVELING_CAPABLE */ -#ifdef SDRAM_PHY_BITSLIPS +#if defined(SDRAM_PHY_BITSLIPS) && defined(SDRAM_PHY_WRITE_LEVELING_CAPABLE) /** * Command "sdram_rst_bitslip" * * Reset write leveling Bitslip * */ -#if defined(CSR_SDRAM_BASE) && defined(CSR_DDRPHY_BASE) +#if defined(CSR_DDRPHY_BASE) static void sdram_rst_bitslip_handler(int nb_params, char **params) { char *c; @@ -327,7 +324,7 @@ define_command(sdram_rst_bitslip, sdram_rst_bitslip_handler, "Reset write leveli * Force write leveling Bitslip * */ -#if defined(CSR_SDRAM_BASE) && defined(CSR_DDRPHY_BASE) +#if defined(CSR_DDRPHY_BASE) static void sdram_force_bitslip_handler(int nb_params, char **params) { char *c; @@ -354,7 +351,7 @@ static void sdram_force_bitslip_handler(int nb_params, char **params) define_command(sdram_force_bitslip, sdram_force_bitslip_handler, "Force write leveling Bitslip", LITEDRAM_CMDS); #endif -#endif +#endif /* defined(SDRAM_PHY_BITSLIPS) && defined(SDRAM_PHY_WRITE_LEVELING_CAPABLE) */ /** * Command "sdram_mr_write" @@ -362,7 +359,6 @@ define_command(sdram_force_bitslip, sdram_force_bitslip_handler, "Force write le * Write SDRAM Mode Register * */ -#if defined(CSR_SDRAM_BASE) static void sdram_mr_write_handler(int nb_params, char **params) { char *c; @@ -389,7 +385,8 @@ static void sdram_mr_write_handler(int nb_params, char **params) sdram_software_control_off(); } define_command(sdram_mr_write, sdram_mr_write_handler, "Write SDRAM Mode Register", LITEDRAM_CMDS); -#endif + +#endif /* CSR_SDRAM_BASE */ /** * Command "sdram_spd" diff --git a/litex/soc/software/liblitedram/accessors.c b/litex/soc/software/liblitedram/accessors.c index fb0c48b52..dba4d3fea 100644 --- a/litex/soc/software/liblitedram/accessors.c +++ b/litex/soc/software/liblitedram/accessors.c @@ -94,7 +94,7 @@ void write_rst_delay(int module) { #endif // defined(SDRAM_PHY_WRITE_LEVELING_CAPABLE) -#if defined(SDRAM_PHY_WRITE_LATENCY_CALIBRATION_CAPABLE) || defined(SDRAM_BITSLIP_CAPABLE) +#if defined(SDRAM_PHY_BITSLIPS) int read_dq_bitslip[SDRAM_PHY_MODULES]; void read_inc_dq_bitslip(int module) { @@ -122,10 +122,7 @@ void write_rst_dq_bitslip(int module) { ddrphy_wdly_dq_bitslip_rst_write(1); } -#endif // defined(SDRAM_PHY_WRITE_LATENCY_CALIBRATION_CAPABLE) || defined(SDRAM_BITSLIP_CAPABLE) - -#if defined(SDRAM_PHY_WRITE_LEVELING_CAPABLE) || defined(SDRAM_PHY_WRITE_LATENCY_CALIBRATION_CAPABLE) || \ -defined(SDRAM_PHY_READ_LEVELING_CAPABLE) || defined(SDRAM_BITSLIP_CAPABLE) +#endif // defined(SDRAM_PHY_BITSLIPS) void sdram_select(int module) { ddrphy_dly_sel_write(1 << module); @@ -152,11 +149,7 @@ void sdram_leveling_action(int module, action_callback action) { sdram_deselect(module); } -#endif // defined(SDRAM_PHY_WRITE_LEVELING_CAPABLE) || defined(SDRAM_PHY_WRITE_LATENCY_CALIBRATION_CAPABLE) || defined(SDRAM_PHY_READ_LEVELING_CAPABLE) ... - #ifdef SDRAM_PHY_WRITE_LEVELING_CAPABLE - -int _sdram_write_leveling_bitslips[16]; int _sdram_write_leveling_dat_delays[16]; void sdram_write_leveling_rst_dat_delay(int module, int show) { @@ -171,6 +164,8 @@ void sdram_write_leveling_force_dat_delay(int module, int taps, int show) { printf("Forcing Dat delay of module %d to %d taps\n", module, taps); } +#if defined(SDRAM_PHY_BITSLIPS) +int _sdram_write_leveling_bitslips[16]; void sdram_write_leveling_rst_bitslip(int module, int show) { _sdram_write_leveling_bitslips[module] = -1; if (show) @@ -182,6 +177,7 @@ void sdram_write_leveling_force_bitslip(int module, int bitslip, int show) { if (show) printf("Forcing Bitslip of module %d to %d\n", module, bitslip); } +#endif // defined(SDRAM_PHY_BITSLIPS) #endif // SDRAM_PHY_WRITE_LEVELING_CAPABLE #endif // defined(CSR_SDRAM_BASE) && defined(CSR_DDRPHY_BASE) diff --git a/litex/soc/software/liblitedram/accessors.h b/litex/soc/software/liblitedram/accessors.h index 7083d2c27..a63c30358 100644 --- a/litex/soc/software/liblitedram/accessors.h +++ b/litex/soc/software/liblitedram/accessors.h @@ -37,7 +37,7 @@ void write_rst_delay(int module); #endif // defined(SDRAM_PHY_WRITE_LEVELING_CAPABLE) -#if defined(SDRAM_PHY_WRITE_LATENCY_CALIBRATION_CAPABLE) || defined(SDRAM_BITSLIP_CAPABLE) +#if defined(SDRAM_PHY_BITSLIPS) extern int read_dq_bitslip[SDRAM_PHY_MODULES]; void read_inc_dq_bitslip(int module); @@ -47,25 +47,22 @@ extern int write_dq_bitslip[SDRAM_PHY_MODULES]; void write_inc_dq_bitslip(int module); void write_rst_dq_bitslip(int module); -#endif // defined(SDRAM_PHY_WRITE_LATENCY_CALIBRATION_CAPABLE) || defined(SDRAM_BITSLIP_CAPABLE) - -#if defined(SDRAM_PHY_WRITE_LEVELING_CAPABLE) || defined(SDRAM_PHY_WRITE_LATENCY_CALIBRATION_CAPABLE) || \ -defined(SDRAM_PHY_READ_LEVELING_CAPABLE) || defined(SDRAM_BITSLIP_CAPABLE) +#endif // defined(SDRAM_PHY_BITSLIPS) void sdram_select(int module); void sdram_deselect(int module); void sdram_leveling_action(int module, action_callback action); -#endif // defined(SDRAM_PHY_WRITE_LEVELING_CAPABLE) || defined(SDRAM_PHY_WRITE_LATENCY_CALIBRATION_CAPABLE) || defined(SDRAM_PHY_READ_LEVELING_CAPABLE) ... - #ifdef SDRAM_PHY_WRITE_LEVELING_CAPABLE -extern int _sdram_write_leveling_bitslips[16]; extern int _sdram_write_leveling_dat_delays[16]; - void sdram_write_leveling_rst_dat_delay(int module, int show); void sdram_write_leveling_force_dat_delay(int module, int taps, int show); + +#if defined(SDRAM_PHY_BITSLIPS) +extern int _sdram_write_leveling_bitslips[16]; void sdram_write_leveling_rst_bitslip(int module, int show); void sdram_write_leveling_force_bitslip(int module, int bitslip, int show); +#endif // defined(SDRAM_PHY_BITSLIPS) #endif // SDRAM_PHY_WRITE_LEVELING_CAPABLE #endif // defined(CSR_SDRAM_BASE) && defined(CSR_DDRPHY_BASE) diff --git a/litex/soc/software/liblitedram/sdram.c b/litex/soc/software/liblitedram/sdram.c index 1bba4b745..a40ec62ec 100644 --- a/litex/soc/software/liblitedram/sdram.c +++ b/litex/soc/software/liblitedram/sdram.c @@ -10,6 +10,7 @@ // License: BSD #include +#ifdef CSR_SDRAM_BASE #include #include @@ -18,9 +19,7 @@ #include #include -#ifdef CSR_SDRAM_BASE #include -#endif #include #include @@ -33,19 +32,17 @@ //#define SDRAM_WRITE_LATENCY_CALIBRATION_DEBUG //#define SDRAM_LEVELING_SCAN_DISPLAY_HEX_DIV 10 -#ifdef CSR_SDRAM_BASE - #ifdef SDRAM_WRITE_LATENCY_CALIBRATION_DEBUG #define SDRAM_WLC_DEBUG 1 #else #define SDRAM_WLC_DEBUG 0 -#endif +#endif // SDRAM_WRITE_LATENCY_CALIBRATION_DEBUG #if SDRAM_PHY_DELAYS > 32 #define MODULO (SDRAM_PHY_DELAYS/32) #else #define MODULO (1) -#endif +#endif // SDRAM_PHY_DELAYS > 32 /*-----------------------------------------------------------------------*/ /* Helpers */ @@ -54,14 +51,13 @@ #define max(x, y) (((x) > (y)) ? (x) : (y)) #define min(x, y) (((x) < (y)) ? (x) : (y)) -__attribute__((unused)) void cdelay(int i) -{ +__attribute__((unused)) void cdelay(int i) { #ifndef CONFIG_BIOS_NO_DELAYS while(i > 0) { __asm__ volatile(CONFIG_CPU_NOP); i--; } -#endif +#endif // CONFIG_BIOS_NO_DELAYS } /*-----------------------------------------------------------------------*/ @@ -81,9 +77,9 @@ int sdram_get_freq(void) { int sdram_get_cl(void) { #ifdef SDRAM_PHY_CL return SDRAM_PHY_CL; -#else +#else // not SDRAM_PHY_CL return -1; -#endif +#endif // SDRAM_PHY_CL } int sdram_get_cwl(void) { @@ -91,7 +87,7 @@ int sdram_get_cwl(void) { return SDRAM_PHY_CWL; #else return -1; -#endif +#endif // SDRAM_PHY_CWL } /*-----------------------------------------------------------------------*/ @@ -99,13 +95,12 @@ int sdram_get_cwl(void) { /*-----------------------------------------------------------------------*/ #ifdef CSR_DDRPHY_BASE - static unsigned char sdram_dfii_get_rdphase(void) { #ifdef CSR_DDRPHY_RDPHASE_ADDR return ddrphy_rdphase_read(); #else return SDRAM_PHY_RDPHASE; -#endif +#endif // CSR_DDRPHY_RDPHASE_ADDR } static unsigned char sdram_dfii_get_wrphase(void) { @@ -113,27 +108,27 @@ static unsigned char sdram_dfii_get_wrphase(void) { return ddrphy_wrphase_read(); #else return SDRAM_PHY_WRPHASE; -#endif +#endif // CSR_DDRPHY_WRPHASE_ADDR } static void sdram_dfii_pix_address_write(unsigned char phase, unsigned int value) { #if (SDRAM_PHY_PHASES > 8) #error "More than 8 DFI phases not supported" -#endif +#endif // (SDRAM_PHY_PHASES > 8) switch (phase) { #if (SDRAM_PHY_PHASES > 4) case 7: sdram_dfii_pi7_address_write(value); break; case 6: sdram_dfii_pi6_address_write(value); break; case 5: sdram_dfii_pi5_address_write(value); break; case 4: sdram_dfii_pi4_address_write(value); break; -#endif +#endif // (SDRAM_PHY_PHASES > 4) #if (SDRAM_PHY_PHASES > 2) case 3: sdram_dfii_pi3_address_write(value); break; case 2: sdram_dfii_pi2_address_write(value); break; -#endif +#endif // (SDRAM_PHY_PHASES > 2) #if (SDRAM_PHY_PHASES > 1) case 1: sdram_dfii_pi1_address_write(value); break; -#endif +#endif // (SDRAM_PHY_PHASES > 1) default: sdram_dfii_pi0_address_write(value); } } @@ -151,21 +146,21 @@ static void sdram_dfii_piwr_address_write(unsigned int value) { static void sdram_dfii_pix_baddress_write(unsigned char phase, unsigned int value) { #if (SDRAM_PHY_PHASES > 8) #error "More than 8 DFI phases not supported" -#endif +#endif // (SDRAM_PHY_PHASES > 8) switch (phase) { #if (SDRAM_PHY_PHASES > 4) case 7: sdram_dfii_pi7_baddress_write(value); break; case 6: sdram_dfii_pi6_baddress_write(value); break; case 5: sdram_dfii_pi5_baddress_write(value); break; case 4: sdram_dfii_pi4_baddress_write(value); break; -#endif +#endif // (SDRAM_PHY_PHASES > 4) #if (SDRAM_PHY_PHASES > 2) case 3: sdram_dfii_pi3_baddress_write(value); break; case 2: sdram_dfii_pi2_baddress_write(value); break; -#endif +#endif // (SDRAM_PHY_PHASES > 2) #if (SDRAM_PHY_PHASES > 1) case 1: sdram_dfii_pi1_baddress_write(value); break; -#endif +#endif // (SDRAM_PHY_PHASES > 1) default: sdram_dfii_pi0_baddress_write(value); } } @@ -183,21 +178,21 @@ static void sdram_dfii_piwr_baddress_write(unsigned int value) { static void command_px(unsigned char phase, unsigned int value) { #if (SDRAM_PHY_PHASES > 8) #error "More than 8 DFI phases not supported" -#endif +#endif // (SDRAM_PHY_PHASES > 8) switch (phase) { #if (SDRAM_PHY_PHASES > 4) case 7: command_p7(value); break; case 6: command_p6(value); break; case 5: command_p5(value); break; case 4: command_p4(value); break; -#endif +#endif // (SDRAM_PHY_PHASES > 4) #if (SDRAM_PHY_PHASES > 2) case 3: command_p3(value); break; case 2: command_p2(value); break; -#endif +#endif // (SDRAM_PHY_PHASES > 2) #if (SDRAM_PHY_PHASES > 1) case 1: command_p1(value); break; -#endif +#endif // (SDRAM_PHY_PHASES > 1) default: command_p0(value); } } @@ -211,8 +206,7 @@ static void command_pwr(unsigned int value) { unsigned char wrphase = sdram_dfii_get_wrphase(); command_px(wrphase, value); } - -#endif +#endif // CSR_DDRPHY_BASE /*-----------------------------------------------------------------------*/ /* Software/Hardware Control */ @@ -221,8 +215,7 @@ static void command_pwr(unsigned int value) { #define DFII_CONTROL_SOFTWARE (DFII_CONTROL_CKE|DFII_CONTROL_ODT|DFII_CONTROL_RESET_N) #define DFII_CONTROL_HARDWARE (DFII_CONTROL_SEL) -void sdram_software_control_on(void) -{ +void sdram_software_control_on(void) { unsigned int previous; previous = sdram_dfii_control_read(); /* Switch DFII to software control */ @@ -234,11 +227,10 @@ void sdram_software_control_on(void) #if CSR_DDRPHY_EN_VTC_ADDR /* Disable Voltage/Temperature compensation */ ddrphy_en_vtc_write(0); -#endif +#endif // CSR_DDRPHY_EN_VTC_ADDR } -void sdram_software_control_off(void) -{ +void sdram_software_control_off(void) { unsigned int previous; previous = sdram_dfii_control_read(); /* Switch DFII to hardware control */ @@ -249,7 +241,7 @@ void sdram_software_control_off(void) #if CSR_DDRPHY_EN_VTC_ADDR /* Enable Voltage/Temperature compensation */ ddrphy_en_vtc_write(1); -#endif +#endif // CSR_DDRPHY_EN_VTC_ADDR } /*-----------------------------------------------------------------------*/ @@ -305,7 +297,7 @@ static void print_scan_errors(unsigned int errors) { printf("%x", errors); #else printf("%d", errors == 0); -#endif +#endif // SDRAM_LEVELING_SCAN_DISPLAY_HEX_DIV } #define READ_CHECK_TEST_PATTERN_MAX_ERRORS (8*SDRAM_PHY_PHASES*DFII_PIX_DATA_BYTES/SDRAM_PHY_MODULES) @@ -346,7 +338,7 @@ static unsigned int sdram_write_read_check_test_pattern(int module, unsigned int #if defined(SDRAM_PHY_ECP5DDRPHY) || defined(SDRAM_PHY_GW2DDRPHY) ddrphy_burstdet_clr_write(1); -#endif +#endif // defined(SDRAM_PHY_ECP5DDRPHY) || defined(SDRAM_PHY_GW2DDRPHY) /* Read/Check pseudo-random sequence */ sdram_dfii_pird_address_write(0); @@ -397,13 +389,13 @@ static unsigned int sdram_write_read_check_test_pattern(int module, unsigned int if (SDRAM_PHY_DQ_DQS_RATIO == 16) errors += popcount(((prs[p][nebo+1] >> ibo) & mask) ^ ((tst[nebo+1] >> ibo) & mask)); -#endif //SDRAM_PHY_XDR == 2 +#endif // SDRAM_PHY_XDR == 2 } #if defined(SDRAM_PHY_ECP5DDRPHY) || defined(SDRAM_PHY_GW2DDRPHY) if (((ddrphy_burstdet_seen_read() >> module) & 0x1) != 1) errors += 1; -#endif +#endif // defined(SDRAM_PHY_ECP5DDRPHY) || defined(SDRAM_PHY_GW2DDRPHY) return errors; } @@ -411,9 +403,7 @@ static unsigned int sdram_write_read_check_test_pattern(int module, unsigned int static int _seed_array[] = {42, 84, 36, 72, 24, 48}; static int _seed_array_length = sizeof(_seed_array) / sizeof(_seed_array[0]); - -static int run_test_pattern(int module) -{ +static int run_test_pattern(int module) { int errors = 0; for (int i = 0; i < _seed_array_length; i++) { errors += sdram_write_read_check_test_pattern(module, _seed_array[i]); @@ -422,8 +412,8 @@ static int run_test_pattern(int module) } static void sdram_leveling_center_module( - int module, int show_short, int show_long, action_callback rst_delay, action_callback inc_delay) -{ + int module, int show_short, int show_long, action_callback rst_delay, action_callback inc_delay) { + int i; int show; int working, last_working; @@ -524,18 +514,17 @@ static void sdram_leveling_center_module( /* Write Leveling */ /*-----------------------------------------------------------------------*/ -int _sdram_tck_taps; - #ifdef SDRAM_PHY_WRITE_LEVELING_CAPABLE +int _sdram_tck_taps; + int _sdram_write_leveling_cmd_scan = 1; int _sdram_write_leveling_cmd_delay = 0; int _sdram_write_leveling_cdly_range_start = -1; int _sdram_write_leveling_cdly_range_end = -1; -static void sdram_write_leveling_on(void) -{ +static void sdram_write_leveling_on(void) { // Flip write leveling bit in the Mode Register, as it is disabled by default sdram_dfii_pi0_address_write(DDRX_MR_WRLVL_RESET ^ (1 << DDRX_MR_WRLVL_BIT)); sdram_dfii_pi0_baddress_write(DDRX_MR_WRLVL_ADDRESS); @@ -545,13 +534,12 @@ static void sdram_write_leveling_on(void) sdram_dfii_pi0_address_write((DDRX_MR_WRLVL_RESET ^ (1 << DDRX_MR_WRLVL_BIT)) ^ 0x2BF8) ; sdram_dfii_pi0_baddress_write(DDRX_MR_WRLVL_ADDRESS ^ 0xF); command_p0(DFII_COMMAND_RAS|DFII_COMMAND_CAS|DFII_COMMAND_WE|DFII_COMMAND_CS); -#endif +#endif // SDRAM_PHY_DDR4_RDIMM ddrphy_wlevel_en_write(1); } -static void sdram_write_leveling_off(void) -{ +static void sdram_write_leveling_off(void) { sdram_dfii_pi0_address_write(DDRX_MR_WRLVL_RESET); sdram_dfii_pi0_baddress_write(DDRX_MR_WRLVL_ADDRESS); command_p0(DFII_COMMAND_RAS|DFII_COMMAND_CAS|DFII_COMMAND_WE|DFII_COMMAND_CS); @@ -560,7 +548,7 @@ static void sdram_write_leveling_off(void) sdram_dfii_pi0_address_write(DDRX_MR_WRLVL_RESET ^ 0x2BF8); sdram_dfii_pi0_baddress_write(DDRX_MR_WRLVL_ADDRESS ^ 0xF); command_p0(DFII_COMMAND_RAS|DFII_COMMAND_CAS|DFII_COMMAND_WE|DFII_COMMAND_CS); -#endif +#endif // SDRAM_PHY_DDR4_RDIMM ddrphy_wlevel_en_write(0); } @@ -583,8 +571,7 @@ void sdram_write_leveling_force_cmd_delay(int taps, int show) { } } -static int sdram_write_leveling_scan(int *delays, int loops, int show) -{ +static int sdram_write_leveling_scan(int *delays, int loops, int show) { int i, j, k; int err_ddrphy_wdly; @@ -630,9 +617,9 @@ static int sdram_write_leveling_scan(int *delays, int loops, int show) module_byte >>= 4 * (i % 2); /* Extract the nibble from the tested module */ if ((module_byte & 0xf) != 0) -#else +#else // SDRAM_PHY_DQ_DQS_RATIO != 4 if (buf[SDRAM_PHY_MODULES-1-i] != 0) -#endif +#endif // SDRAM_PHY_DQ_DQS_RATIO == 4 one_count++; else zero_count++; @@ -693,11 +680,11 @@ static int sdram_write_leveling_scan(int *delays, int loops, int show) ((one_window_best_start) > 0 && (one_window_best_count > 0)) || /* Start of 1s window indirectly seen before 0. */ ((one_window_best_start == 0) && (one_window_best_count > _sdram_tck_taps/4)) - ){ + ) { #if SDRAM_PHY_DELAYS > 32 /* Ensure write delay is just before transition */ one_window_start -= min(one_window_start, 16); -#endif +#endif // SDRAM_PHY_DELAYS > 32 delays[i] = one_window_best_start; /* Configure write delay */ @@ -725,14 +712,14 @@ static int sdram_write_leveling_scan(int *delays, int loops, int show) return ok; } -static void sdram_write_leveling_find_cmd_delay(unsigned int *best_error, unsigned int *best_count, int *best_cdly, - int cdly_start, int cdly_stop, int cdly_step) -{ +static void sdram_write_leveling_find_cmd_delay( + unsigned int *best_error, unsigned int *best_count, int *best_cdly, + int cdly_start, int cdly_stop, int cdly_step) { int cdly; int delays[SDRAM_PHY_MODULES]; #ifndef SDRAM_WRITE_LEVELING_CMD_DELAY_DEBUG int ok; -#endif +#endif // SDRAM_WRITE_LEVELING_CMD_DELAY_DEBUG /* Scan through the range */ sdram_rst_clock_delay(); @@ -747,7 +734,7 @@ static void sdram_write_leveling_find_cmd_delay(unsigned int *best_error, unsign sdram_write_leveling_scan(delays, 8, 1); #else ok = sdram_write_leveling_scan(delays, 8, 0); -#endif +#endif // SDRAM_WRITE_LEVELING_CMD_DELAY_DEBUG /* Use the mean of delays for error calulation */ int delay_mean = 0; int delay_count = 0; @@ -777,12 +764,11 @@ static void sdram_write_leveling_find_cmd_delay(unsigned int *best_error, unsign printf("Delay mean: %d/256, ideal: %d/256\n", delay_mean, ideal_delay); #else printf("%d", ok); -#endif +#endif // SDRAM_WRITE_LEVELING_CMD_DELAY_DEBUG } } -int sdram_write_leveling(void) -{ +int sdram_write_leveling(void) { int delays[SDRAM_PHY_MODULES]; unsigned int best_error = ~0u; unsigned int best_count = 0; @@ -852,16 +838,16 @@ int sdram_write_leveling(void) return best_cdly >= 0; } - #endif /* SDRAM_PHY_WRITE_LEVELING_CAPABLE */ /*-----------------------------------------------------------------------*/ /* Read Leveling */ /*-----------------------------------------------------------------------*/ -static unsigned int sdram_read_leveling_scan_module(int module, int bitslip, int show) -{ - const unsigned int max_errors = _seed_array_length * READ_CHECK_TEST_PATTERN_MAX_ERRORS; +#if defined(SDRAM_PHY_WRITE_DQ_DQS_TRAINING_CAPABLE) || defined(SDRAM_PHY_WRITE_LATENCY_CALIBRATION_CAPABLE) || defined(SDRAM_PHY_READ_LEVELING_CAPABLE) + +static unsigned int sdram_read_leveling_scan_module(int module, int bitslip, int show) { + const unsigned int max_errors = _seed_array_length*READ_CHECK_TEST_PATTERN_MAX_ERRORS; int i; unsigned int score; unsigned int errors; @@ -889,16 +875,11 @@ static unsigned int sdram_read_leveling_scan_module(int module, int bitslip, int return score; } -#endif /* CSR_DDRPHY_BASE */ +#endif // defined(SDRAM_PHY_WRITE_DQ_DQS_TRAINING_CAPABLE) || defined(SDRAM_PHY_WRITE_LATENCY_CALIBRATION_CAPABLE) || defined(SDRAM_PHY_READ_LEVELING_CAPABLE) -#endif /* CSR_SDRAM_BASE */ +#ifdef SDRAM_PHY_READ_LEVELING_CAPABLE -#ifdef CSR_SDRAM_BASE - -#if defined(SDRAM_PHY_WRITE_LEVELING_CAPABLE) || defined(SDRAM_PHY_READ_LEVELING_CAPABLE) - -void sdram_read_leveling(void) -{ +void sdram_read_leveling(void) { int module; int bitslip; unsigned int score; @@ -940,6 +921,10 @@ void sdram_read_leveling(void) } } +#endif // SDRAM_PHY_READ_LEVELING_CAPABLE + +#endif /* CSR_DDRPHY_BASE */ + /*-----------------------------------------------------------------------*/ /* Write latency calibration */ /*-----------------------------------------------------------------------*/ @@ -996,7 +981,7 @@ static void sdram_write_latency_calibration(void) { bitslip = _sdram_write_leveling_bitslips[module]; #else bitslip = best_bitslip; - #endif + #endif // SDRAM_PHY_WRITE_LEVELING_CAPABLE if (bitslip == -1) printf("m%d:- ", module); else @@ -1014,7 +999,7 @@ static void sdram_write_latency_calibration(void) { printf("\n"); } -#endif +#endif // SDRAM_PHY_WRITE_LATENCY_CALIBRATION_CAPABLE /*-----------------------------------------------------------------------*/ /* Write DQ-DQS training */ @@ -1022,8 +1007,7 @@ static void sdram_write_latency_calibration(void) { #ifdef SDRAM_PHY_WRITE_DQ_DQS_TRAINING_CAPABLE -static void sdram_read_leveling_best_bitslip(int module) -{ +static void sdram_read_leveling_best_bitslip(int module) { unsigned int score; int bitslip; int best_bitslip = 0; @@ -1051,8 +1035,7 @@ static void sdram_read_leveling_best_bitslip(int module) read_rst_dq_delay, read_inc_dq_delay); } -static void sdram_write_dq_dqs_training(void) -{ +static void sdram_write_dq_dqs_training(void) { int module; for(module=0; module 0 printf("\nError stats:\n"); sdram_debug_error_stats(); -#endif +#endif // defined(SDRAM_DEBUG_STATS_NUM_RUNS) && SDRAM_DEBUG_STATS_NUM_RUNS > 0 #ifdef SDRAM_DEBUG_READBACK_MEM_ADDR printf("\nReadback:\n"); sdram_debug_readback(); -#endif +#endif // SDRAM_DEBUG_READBACK_MEM_ADDR } -#endif +#endif // SDRAM_DEBUG -#endif +#endif // CSR_SDRAM_BASE From 68877742b17d1624f557c46d281e5b17e199419c Mon Sep 17 00:00:00 2001 From: Maciej Dudek Date: Wed, 25 Jan 2023 22:56:25 +0100 Subject: [PATCH 9/9] liblitedram: Add ability to access and adjust delays per DQ line Signed-off-by: Maciej Dudek --- litex/soc/software/liblitedram/accessors.c | 20 +- litex/soc/software/liblitedram/accessors.h | 6 +- litex/soc/software/liblitedram/sdram.c | 464 +++++++++++---------- 3 files changed, 270 insertions(+), 220 deletions(-) diff --git a/litex/soc/software/liblitedram/accessors.c b/litex/soc/software/liblitedram/accessors.c index dba4d3fea..47c3b733f 100644 --- a/litex/soc/software/liblitedram/accessors.c +++ b/litex/soc/software/liblitedram/accessors.c @@ -124,11 +124,16 @@ void write_rst_dq_bitslip(int module) { #endif // defined(SDRAM_PHY_BITSLIPS) -void sdram_select(int module) { +void sdram_select(int module, int dq_line) { ddrphy_dly_sel_write(1 << module); + +#ifdef SDRAM_DELAY_PER_DQ + /* Select DQ line */ + ddrphy_dq_dly_sel_write(1 << dq_line); +#endif } -void sdram_deselect(int module) { +void sdram_deselect(int module, int dq_line) { ddrphy_dly_sel_write(0); #if defined(SDRAM_PHY_ECP5DDRPHY) || defined(SDRAM_PHY_GW2DDRPHY) @@ -136,17 +141,22 @@ void sdram_deselect(int module) { ddrphy_dly_sel_write(0xff); ddrphy_dly_sel_write(0); #endif //SDRAM_PHY_ECP5DDRPHY + +#ifdef SDRAM_DELAY_PER_DQ + /* Un-select DQ line */ + ddrphy_dq_dly_sel_write(0); +#endif } -void sdram_leveling_action(int module, action_callback action) { +void sdram_leveling_action(int module, int dq_line, action_callback action) { /* Select module */ - sdram_select(module); + sdram_select(module, dq_line); /* Action */ action(module); /* Un-select module */ - sdram_deselect(module); + sdram_deselect(module, dq_line); } #ifdef SDRAM_PHY_WRITE_LEVELING_CAPABLE diff --git a/litex/soc/software/liblitedram/accessors.h b/litex/soc/software/liblitedram/accessors.h index a63c30358..b411493a5 100644 --- a/litex/soc/software/liblitedram/accessors.h +++ b/litex/soc/software/liblitedram/accessors.h @@ -49,9 +49,9 @@ void write_rst_dq_bitslip(int module); #endif // defined(SDRAM_PHY_BITSLIPS) -void sdram_select(int module); -void sdram_deselect(int module); -void sdram_leveling_action(int module, action_callback action); +void sdram_select(int module, int dq_line); +void sdram_deselect(int module, int dq_line); +void sdram_leveling_action(int module, int dq_line, action_callback action); #ifdef SDRAM_PHY_WRITE_LEVELING_CAPABLE extern int _sdram_write_leveling_dat_delays[16]; diff --git a/litex/soc/software/liblitedram/sdram.c b/litex/soc/software/liblitedram/sdram.c index a40ec62ec..06d01c35e 100644 --- a/litex/soc/software/liblitedram/sdram.c +++ b/litex/soc/software/liblitedram/sdram.c @@ -38,6 +38,12 @@ #define SDRAM_WLC_DEBUG 0 #endif // SDRAM_WRITE_LATENCY_CALIBRATION_DEBUG +#ifdef SDRAM_DELAY_PER_DQ +#define DQ_COUNT SDRAM_PHY_DQ_DQS_RATIO +#else +#define DQ_COUNT 1 +#endif + #if SDRAM_PHY_DELAYS > 32 #define MODULO (SDRAM_PHY_DELAYS/32) #else @@ -303,7 +309,7 @@ static void print_scan_errors(unsigned int errors) { #define READ_CHECK_TEST_PATTERN_MAX_ERRORS (8*SDRAM_PHY_PHASES*DFII_PIX_DATA_BYTES/SDRAM_PHY_MODULES) #define MODULE_BITMASK ((1<= SDRAM_PHY_DELAYS) break; - sdram_leveling_action(module, inc_delay); + sdram_leveling_action(module, dq_line, inc_delay); } delay_max = delay_min; cur_delay_min = delay_min; /* Find largest working delay range */ while(1) { - errors = run_test_pattern(module); + errors = run_test_pattern(module, dq_line); working = errors == 0; show = show_long && (delay%MODULO == 0); if (show) @@ -468,7 +483,7 @@ static void sdram_leveling_center_module( delay++; if(delay >= SDRAM_PHY_DELAYS) break; - sdram_leveling_action(module, inc_delay); + sdram_leveling_action(module, dq_line, inc_delay); } if(delay_max < 0) { delay_max = delay; @@ -494,15 +509,15 @@ static void sdram_leveling_center_module( int retries = 8; /* Do N configs/checks and give up if failing */ while (retries > 0) { /* Set delay. */ - sdram_leveling_action(module, rst_delay); + sdram_leveling_action(module, dq_line, rst_delay); cdelay(100); for(i = 0; i < delay_mid; i++) { - sdram_leveling_action(module, inc_delay); + sdram_leveling_action(module, dq_line, inc_delay); cdelay(100); } /* Check */ - errors = run_test_pattern(module); + errors = run_test_pattern(module, dq_line); if (errors == 0) break; retries--; @@ -572,7 +587,7 @@ void sdram_write_leveling_force_cmd_delay(int taps, int show) { } static int sdram_write_leveling_scan(int *delays, int loops, int show) { - int i, j, k; + int i, j, k, dq_line; int err_ddrphy_wdly; @@ -591,113 +606,119 @@ static int sdram_write_leveling_scan(int *delays, int loops, int show) { sdram_write_leveling_on(); cdelay(100); for(i=0;i>= 4 * (i % 2); - /* Extract the nibble from the tested module */ - if ((module_byte & 0xf) != 0) -#else // SDRAM_PHY_DQ_DQS_RATIO != 4 - if (buf[SDRAM_PHY_MODULES-1-i] != 0) -#endif // SDRAM_PHY_DQ_DQS_RATIO == 4 - one_count++; - else - zero_count++; - } - if (one_count > zero_count) - taps_scan[j] = 1; - else - taps_scan[j] = 0; - if (show_iter) - printf("%d", taps_scan[j]); - sdram_leveling_action(i, write_inc_delay); + /* Reset delay */ + sdram_leveling_action(i, dq_line, write_rst_delay); cdelay(100); - } - if (show) - printf("|"); - /* Find longer 1 window and set delay at the 0/1 transition */ - one_window_active = 0; - one_window_start = 0; - one_window_count = 0; - one_window_best_start = 0; - one_window_best_count = -1; - delays[i] = -1; - for(j=0;j one_window_best_count) { - one_window_best_start = one_window_start; - one_window_best_count = one_window_count; + /* Scan write delay taps */ + for(j=0;j>= 4 * (i % 2); + /* Extract the nibble from the tested module */ + if ((module_byte & 0xf) != 0) +#else // SDRAM_PHY_DQ_DQS_RATIO != 4 + if (buf[SDRAM_PHY_MODULES-1-i] != 0) +#endif // SDRAM_PHY_DQ_DQS_RATIO == 4 + one_count++; + else + zero_count++; + } + if (one_count > zero_count) + taps_scan[j] = 1; + else + taps_scan[j] = 0; + if (show_iter) + printf("%d", taps_scan[j]); + sdram_leveling_action(i, dq_line, write_inc_delay); + cdelay(100); + } + if (show) + printf("|"); + + /* Find longer 1 window and set delay at the 0/1 transition */ + one_window_active = 0; + one_window_start = 0; + one_window_count = 0; + one_window_best_start = 0; + one_window_best_count = -1; + delays[i] = -1; + for(j=0;j one_window_best_count) { + one_window_best_start = one_window_start; + one_window_best_count = one_window_count; + } + } + } else { + if (j != err_ddrphy_wdly && taps_scan[j]) { + one_window_active = 1; + one_window_start = j; } } - } else { - if (j != err_ddrphy_wdly && taps_scan[j]) { - one_window_active = 1; - one_window_start = j; + } + + /* Reset delay */ + sdram_leveling_action(i, dq_line, write_rst_delay); + cdelay(100); + + /* Use forced delay if configured */ + if (_sdram_write_leveling_dat_delays[i] >= 0) { + delays[i] = _sdram_write_leveling_dat_delays[i]; + + /* Configure write delay */ + for(j=0; j 0 && (one_window_best_count > 0)) || + /* Start of 1s window indirectly seen before 0. */ + ((one_window_best_start == 0) && (one_window_best_count > _sdram_tck_taps/4)) + ) { +#if SDRAM_PHY_DELAYS > 32 + /* Ensure write delay is just before transition */ + one_window_start -= min(one_window_start, 16); +#endif // SDRAM_PHY_DELAYS > 32 + delays[i] = one_window_best_start; + + /* Configure write delay */ + for(j=0; j= 0) { - delays[i] = _sdram_write_leveling_dat_delays[i]; - - /* Configure write delay */ - for(j=0; j 0 && (one_window_best_count > 0)) || - /* Start of 1s window indirectly seen before 0. */ - ((one_window_best_start == 0) && (one_window_best_count > _sdram_tck_taps/4)) - ) { -#if SDRAM_PHY_DELAYS > 32 - /* Ensure write delay is just before transition */ - one_window_start -= min(one_window_start, 16); -#endif // SDRAM_PHY_DELAYS > 32 - delays[i] = one_window_best_start; - - /* Configure write delay */ - for(j=0; j best_score) { - best_bitslip = bitslip; - best_score = score; + for (dq_line = 0; dq_line < DQ_COUNT; dq_line++) { + /* Scan possible read windows */ + best_score = 0; + best_bitslip = 0; + sdram_leveling_action(module, dq_line, read_rst_dq_bitslip); + for(bitslip=0; bitslip best_score) { + best_bitslip = bitslip; + best_score = score; + } + /* Exit */ + if (bitslip == SDRAM_PHY_BITSLIPS-1) + break; + /* Increment bitslip */ + sdram_leveling_action(module, dq_line, read_inc_dq_bitslip); } - /* Exit */ - if (bitslip == SDRAM_PHY_BITSLIPS-1) - break; - /* Increment bitslip */ - sdram_leveling_action(module, read_inc_dq_bitslip); + + /* Select best read window */ +#ifdef SDRAM_DELAY_PER_DQ + printf(" best: m%d, b%02d, dq_line%d ", module, best_bitslip, dq_line); +#else + printf(" best: m%d, b%02d ", module, best_bitslip); +#endif // SDRAM_DELAY_PER_DQ + sdram_leveling_action(module, dq_line, read_rst_dq_bitslip); + for (bitslip=0; bitslip score ? subscore : score; + /* Increment bitslip */ + sdram_leveling_action(module, dq_line, read_inc_dq_bitslip); + } + if (score > best_score) { + best_bitslip = bitslip; + best_score = score; + } + } + +#ifdef SDRAM_PHY_WRITE_LEVELING_CAPABLE + if (_sdram_write_leveling_bitslips[module] < 0) + bitslip = best_bitslip; + else + bitslip = _sdram_write_leveling_bitslips[module]; +#else + bitslip = best_bitslip; +#endif // SDRAM_PHY_WRITE_LEVELING_CAPABLE + if (bitslip == -1) + printf("m%d:- ", module); + else +#ifdef SDRAM_DELAY_PER_DQ + printf("m%d dq%d:%d ", module, dq_line, bitslip); +#else + printf("m%d:%d ", module, bitslip); +#endif // SDRAM_DELAY_PER_DQ + if (SDRAM_WLC_DEBUG) - printf("m%d wb%02d:\n", module, bitslip); + printf("\n"); /* Reset bitslip */ - sdram_leveling_action(module, write_rst_dq_bitslip); + sdram_leveling_action(module, dq_line, write_rst_dq_bitslip); for (i=0; i score ? subscore : score; - /* Increment bitslip */ - sdram_leveling_action(module, read_inc_dq_bitslip); - } - if (score > best_score) { - best_bitslip = bitslip; - best_score = score; + sdram_leveling_action(module, dq_line, write_inc_dq_bitslip); } } - - #ifdef SDRAM_PHY_WRITE_LEVELING_CAPABLE - if (_sdram_write_leveling_bitslips[module] < 0) - bitslip = best_bitslip; - else - bitslip = _sdram_write_leveling_bitslips[module]; - #else - bitslip = best_bitslip; - #endif // SDRAM_PHY_WRITE_LEVELING_CAPABLE - if (bitslip == -1) - printf("m%d:- ", module); - else - printf("m%d:%d ", module, bitslip); - - if (SDRAM_WLC_DEBUG) - printf("\n"); - - /* Reset bitslip */ - sdram_leveling_action(module, write_rst_dq_bitslip); - for (i=0; i best_score) { best_bitslip = bitslip; best_score = score; } if (bitslip == SDRAM_PHY_BITSLIPS-1) break; - sdram_leveling_action(module, read_inc_dq_bitslip); + sdram_leveling_action(module, dq_line, read_inc_dq_bitslip); } /* Select best read window and re-center it */ - sdram_leveling_action(module, read_rst_dq_bitslip); + sdram_leveling_action(module, dq_line, read_rst_dq_bitslip); for (bitslip=0; bitslip