diff --git a/litex/soc/software/liblitedram/accessors.c b/litex/soc/software/liblitedram/accessors.c index dba4d3fea..47c3b733f 100644 --- a/litex/soc/software/liblitedram/accessors.c +++ b/litex/soc/software/liblitedram/accessors.c @@ -124,11 +124,16 @@ void write_rst_dq_bitslip(int module) { #endif // defined(SDRAM_PHY_BITSLIPS) -void sdram_select(int module) { +void sdram_select(int module, int dq_line) { ddrphy_dly_sel_write(1 << module); + +#ifdef SDRAM_DELAY_PER_DQ + /* Select DQ line */ + ddrphy_dq_dly_sel_write(1 << dq_line); +#endif } -void sdram_deselect(int module) { +void sdram_deselect(int module, int dq_line) { ddrphy_dly_sel_write(0); #if defined(SDRAM_PHY_ECP5DDRPHY) || defined(SDRAM_PHY_GW2DDRPHY) @@ -136,17 +141,22 @@ void sdram_deselect(int module) { ddrphy_dly_sel_write(0xff); ddrphy_dly_sel_write(0); #endif //SDRAM_PHY_ECP5DDRPHY + +#ifdef SDRAM_DELAY_PER_DQ + /* Un-select DQ line */ + ddrphy_dq_dly_sel_write(0); +#endif } -void sdram_leveling_action(int module, action_callback action) { +void sdram_leveling_action(int module, int dq_line, action_callback action) { /* Select module */ - sdram_select(module); + sdram_select(module, dq_line); /* Action */ action(module); /* Un-select module */ - sdram_deselect(module); + sdram_deselect(module, dq_line); } #ifdef SDRAM_PHY_WRITE_LEVELING_CAPABLE diff --git a/litex/soc/software/liblitedram/accessors.h b/litex/soc/software/liblitedram/accessors.h index a63c30358..b411493a5 100644 --- a/litex/soc/software/liblitedram/accessors.h +++ b/litex/soc/software/liblitedram/accessors.h @@ -49,9 +49,9 @@ void write_rst_dq_bitslip(int module); #endif // defined(SDRAM_PHY_BITSLIPS) -void sdram_select(int module); -void sdram_deselect(int module); -void sdram_leveling_action(int module, action_callback action); +void sdram_select(int module, int dq_line); +void sdram_deselect(int module, int dq_line); +void sdram_leveling_action(int module, int dq_line, action_callback action); #ifdef SDRAM_PHY_WRITE_LEVELING_CAPABLE extern int _sdram_write_leveling_dat_delays[16]; diff --git a/litex/soc/software/liblitedram/sdram.c b/litex/soc/software/liblitedram/sdram.c index a40ec62ec..06d01c35e 100644 --- a/litex/soc/software/liblitedram/sdram.c +++ b/litex/soc/software/liblitedram/sdram.c @@ -38,6 +38,12 @@ #define SDRAM_WLC_DEBUG 0 #endif // SDRAM_WRITE_LATENCY_CALIBRATION_DEBUG +#ifdef SDRAM_DELAY_PER_DQ +#define DQ_COUNT SDRAM_PHY_DQ_DQS_RATIO +#else +#define DQ_COUNT 1 +#endif + #if SDRAM_PHY_DELAYS > 32 #define MODULO (SDRAM_PHY_DELAYS/32) #else @@ -303,7 +309,7 @@ static void print_scan_errors(unsigned int errors) { #define READ_CHECK_TEST_PATTERN_MAX_ERRORS (8*SDRAM_PHY_PHASES*DFII_PIX_DATA_BYTES/SDRAM_PHY_MODULES) #define MODULE_BITMASK ((1<= SDRAM_PHY_DELAYS) break; - sdram_leveling_action(module, inc_delay); + sdram_leveling_action(module, dq_line, inc_delay); } delay_max = delay_min; cur_delay_min = delay_min; /* Find largest working delay range */ while(1) { - errors = run_test_pattern(module); + errors = run_test_pattern(module, dq_line); working = errors == 0; show = show_long && (delay%MODULO == 0); if (show) @@ -468,7 +483,7 @@ static void sdram_leveling_center_module( delay++; if(delay >= SDRAM_PHY_DELAYS) break; - sdram_leveling_action(module, inc_delay); + sdram_leveling_action(module, dq_line, inc_delay); } if(delay_max < 0) { delay_max = delay; @@ -494,15 +509,15 @@ static void sdram_leveling_center_module( int retries = 8; /* Do N configs/checks and give up if failing */ while (retries > 0) { /* Set delay. */ - sdram_leveling_action(module, rst_delay); + sdram_leveling_action(module, dq_line, rst_delay); cdelay(100); for(i = 0; i < delay_mid; i++) { - sdram_leveling_action(module, inc_delay); + sdram_leveling_action(module, dq_line, inc_delay); cdelay(100); } /* Check */ - errors = run_test_pattern(module); + errors = run_test_pattern(module, dq_line); if (errors == 0) break; retries--; @@ -572,7 +587,7 @@ void sdram_write_leveling_force_cmd_delay(int taps, int show) { } static int sdram_write_leveling_scan(int *delays, int loops, int show) { - int i, j, k; + int i, j, k, dq_line; int err_ddrphy_wdly; @@ -591,113 +606,119 @@ static int sdram_write_leveling_scan(int *delays, int loops, int show) { sdram_write_leveling_on(); cdelay(100); for(i=0;i>= 4 * (i % 2); - /* Extract the nibble from the tested module */ - if ((module_byte & 0xf) != 0) -#else // SDRAM_PHY_DQ_DQS_RATIO != 4 - if (buf[SDRAM_PHY_MODULES-1-i] != 0) -#endif // SDRAM_PHY_DQ_DQS_RATIO == 4 - one_count++; - else - zero_count++; - } - if (one_count > zero_count) - taps_scan[j] = 1; - else - taps_scan[j] = 0; - if (show_iter) - printf("%d", taps_scan[j]); - sdram_leveling_action(i, write_inc_delay); + /* Reset delay */ + sdram_leveling_action(i, dq_line, write_rst_delay); cdelay(100); - } - if (show) - printf("|"); - /* Find longer 1 window and set delay at the 0/1 transition */ - one_window_active = 0; - one_window_start = 0; - one_window_count = 0; - one_window_best_start = 0; - one_window_best_count = -1; - delays[i] = -1; - for(j=0;j one_window_best_count) { - one_window_best_start = one_window_start; - one_window_best_count = one_window_count; + /* Scan write delay taps */ + for(j=0;j>= 4 * (i % 2); + /* Extract the nibble from the tested module */ + if ((module_byte & 0xf) != 0) +#else // SDRAM_PHY_DQ_DQS_RATIO != 4 + if (buf[SDRAM_PHY_MODULES-1-i] != 0) +#endif // SDRAM_PHY_DQ_DQS_RATIO == 4 + one_count++; + else + zero_count++; + } + if (one_count > zero_count) + taps_scan[j] = 1; + else + taps_scan[j] = 0; + if (show_iter) + printf("%d", taps_scan[j]); + sdram_leveling_action(i, dq_line, write_inc_delay); + cdelay(100); + } + if (show) + printf("|"); + + /* Find longer 1 window and set delay at the 0/1 transition */ + one_window_active = 0; + one_window_start = 0; + one_window_count = 0; + one_window_best_start = 0; + one_window_best_count = -1; + delays[i] = -1; + for(j=0;j one_window_best_count) { + one_window_best_start = one_window_start; + one_window_best_count = one_window_count; + } + } + } else { + if (j != err_ddrphy_wdly && taps_scan[j]) { + one_window_active = 1; + one_window_start = j; } } - } else { - if (j != err_ddrphy_wdly && taps_scan[j]) { - one_window_active = 1; - one_window_start = j; + } + + /* Reset delay */ + sdram_leveling_action(i, dq_line, write_rst_delay); + cdelay(100); + + /* Use forced delay if configured */ + if (_sdram_write_leveling_dat_delays[i] >= 0) { + delays[i] = _sdram_write_leveling_dat_delays[i]; + + /* Configure write delay */ + for(j=0; j 0 && (one_window_best_count > 0)) || + /* Start of 1s window indirectly seen before 0. */ + ((one_window_best_start == 0) && (one_window_best_count > _sdram_tck_taps/4)) + ) { +#if SDRAM_PHY_DELAYS > 32 + /* Ensure write delay is just before transition */ + one_window_start -= min(one_window_start, 16); +#endif // SDRAM_PHY_DELAYS > 32 + delays[i] = one_window_best_start; + + /* Configure write delay */ + for(j=0; j= 0) { - delays[i] = _sdram_write_leveling_dat_delays[i]; - - /* Configure write delay */ - for(j=0; j 0 && (one_window_best_count > 0)) || - /* Start of 1s window indirectly seen before 0. */ - ((one_window_best_start == 0) && (one_window_best_count > _sdram_tck_taps/4)) - ) { -#if SDRAM_PHY_DELAYS > 32 - /* Ensure write delay is just before transition */ - one_window_start -= min(one_window_start, 16); -#endif // SDRAM_PHY_DELAYS > 32 - delays[i] = one_window_best_start; - - /* Configure write delay */ - for(j=0; j best_score) { - best_bitslip = bitslip; - best_score = score; + for (dq_line = 0; dq_line < DQ_COUNT; dq_line++) { + /* Scan possible read windows */ + best_score = 0; + best_bitslip = 0; + sdram_leveling_action(module, dq_line, read_rst_dq_bitslip); + for(bitslip=0; bitslip best_score) { + best_bitslip = bitslip; + best_score = score; + } + /* Exit */ + if (bitslip == SDRAM_PHY_BITSLIPS-1) + break; + /* Increment bitslip */ + sdram_leveling_action(module, dq_line, read_inc_dq_bitslip); } - /* Exit */ - if (bitslip == SDRAM_PHY_BITSLIPS-1) - break; - /* Increment bitslip */ - sdram_leveling_action(module, read_inc_dq_bitslip); + + /* Select best read window */ +#ifdef SDRAM_DELAY_PER_DQ + printf(" best: m%d, b%02d, dq_line%d ", module, best_bitslip, dq_line); +#else + printf(" best: m%d, b%02d ", module, best_bitslip); +#endif // SDRAM_DELAY_PER_DQ + sdram_leveling_action(module, dq_line, read_rst_dq_bitslip); + for (bitslip=0; bitslip score ? subscore : score; + /* Increment bitslip */ + sdram_leveling_action(module, dq_line, read_inc_dq_bitslip); + } + if (score > best_score) { + best_bitslip = bitslip; + best_score = score; + } + } + +#ifdef SDRAM_PHY_WRITE_LEVELING_CAPABLE + if (_sdram_write_leveling_bitslips[module] < 0) + bitslip = best_bitslip; + else + bitslip = _sdram_write_leveling_bitslips[module]; +#else + bitslip = best_bitslip; +#endif // SDRAM_PHY_WRITE_LEVELING_CAPABLE + if (bitslip == -1) + printf("m%d:- ", module); + else +#ifdef SDRAM_DELAY_PER_DQ + printf("m%d dq%d:%d ", module, dq_line, bitslip); +#else + printf("m%d:%d ", module, bitslip); +#endif // SDRAM_DELAY_PER_DQ + if (SDRAM_WLC_DEBUG) - printf("m%d wb%02d:\n", module, bitslip); + printf("\n"); /* Reset bitslip */ - sdram_leveling_action(module, write_rst_dq_bitslip); + sdram_leveling_action(module, dq_line, write_rst_dq_bitslip); for (i=0; i score ? subscore : score; - /* Increment bitslip */ - sdram_leveling_action(module, read_inc_dq_bitslip); - } - if (score > best_score) { - best_bitslip = bitslip; - best_score = score; + sdram_leveling_action(module, dq_line, write_inc_dq_bitslip); } } - - #ifdef SDRAM_PHY_WRITE_LEVELING_CAPABLE - if (_sdram_write_leveling_bitslips[module] < 0) - bitslip = best_bitslip; - else - bitslip = _sdram_write_leveling_bitslips[module]; - #else - bitslip = best_bitslip; - #endif // SDRAM_PHY_WRITE_LEVELING_CAPABLE - if (bitslip == -1) - printf("m%d:- ", module); - else - printf("m%d:%d ", module, bitslip); - - if (SDRAM_WLC_DEBUG) - printf("\n"); - - /* Reset bitslip */ - sdram_leveling_action(module, write_rst_dq_bitslip); - for (i=0; i best_score) { best_bitslip = bitslip; best_score = score; } if (bitslip == SDRAM_PHY_BITSLIPS-1) break; - sdram_leveling_action(module, read_inc_dq_bitslip); + sdram_leveling_action(module, dq_line, read_inc_dq_bitslip); } /* Select best read window and re-center it */ - sdram_leveling_action(module, read_rst_dq_bitslip); + sdram_leveling_action(module, dq_line, read_rst_dq_bitslip); for (bitslip=0; bitslip