Hardware
/
litex
mirror of https://github.com/enjoy-digital/litex.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

liblitedram: Add ability to access and adjust delays per DQ line 

Signed-off-by: Maciej Dudek <mdudek@antmicro.com>
This commit is contained in:
Maciej Dudek 2023-01-25 22:56:25 +01:00 committed by Michal Sieron
parent 4c605020f0
commit 68877742b1
3 changed files with 270 additions and 220 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
litex/soc/software/liblitedram/accessors.c
View File

@ -124,11 +124,16 @@ void write_rst_dq_bitslip(int module) {
#endif // defined(SDRAM_PHY_BITSLIPS)
void sdram_select(int module) {
void sdram_select(int module, int dq_line) {
ddrphy_dly_sel_write(1 << module);
#ifdef SDRAM_DELAY_PER_DQ
/* Select DQ line */
ddrphy_dq_dly_sel_write(1 << dq_line);
#endif
}
void sdram_deselect(int module) {
void sdram_deselect(int module, int dq_line) {
ddrphy_dly_sel_write(0);
#if defined(SDRAM_PHY_ECP5DDRPHY) || defined(SDRAM_PHY_GW2DDRPHY)
@ -136,17 +141,22 @@ void sdram_deselect(int module) {
ddrphy_dly_sel_write(0xff);
ddrphy_dly_sel_write(0);
#endif //SDRAM_PHY_ECP5DDRPHY
#ifdef SDRAM_DELAY_PER_DQ
/* Un-select DQ line */
ddrphy_dq_dly_sel_write(0);
#endif
}
void sdram_leveling_action(int module, action_callback action) {
void sdram_leveling_action(int module, int dq_line, action_callback action) {
/* Select module */
sdram_select(module);
sdram_select(module, dq_line);
/* Action */
action(module);
/* Un-select module */
sdram_deselect(module);
sdram_deselect(module, dq_line);
}
#ifdef SDRAM_PHY_WRITE_LEVELING_CAPABLE

6
litex/soc/software/liblitedram/accessors.h
View File

@ -49,9 +49,9 @@ void write_rst_dq_bitslip(int module);
#endif // defined(SDRAM_PHY_BITSLIPS)
void sdram_select(int module);
void sdram_deselect(int module);
void sdram_leveling_action(int module, action_callback action);
void sdram_select(int module, int dq_line);
void sdram_deselect(int module, int dq_line);
void sdram_leveling_action(int module, int dq_line, action_callback action);
#ifdef SDRAM_PHY_WRITE_LEVELING_CAPABLE
extern int _sdram_write_leveling_dat_delays[16];

142
litex/soc/software/liblitedram/sdram.c
View File

@ -38,6 +38,12 @@
#define SDRAM_WLC_DEBUG 0
#endif // SDRAM_WRITE_LATENCY_CALIBRATION_DEBUG
#ifdef SDRAM_DELAY_PER_DQ
#define DQ_COUNT SDRAM_PHY_DQ_DQS_RATIO
#else
#define DQ_COUNT 1
#endif
#if SDRAM_PHY_DELAYS > 32
#define MODULO (SDRAM_PHY_DELAYS/32)
#else
@ -303,7 +309,7 @@ static void print_scan_errors(unsigned int errors) {
#define READ_CHECK_TEST_PATTERN_MAX_ERRORS (8*SDRAM_PHY_PHASES*DFII_PIX_DATA_BYTES/SDRAM_PHY_MODULES)
#define MODULE_BITMASK ((1<<SDRAM_PHY_DQ_DQS_RATIO)-1)
static unsigned int sdram_write_read_check_test_pattern(int module, unsigned int seed) {
static unsigned int sdram_write_read_check_test_pattern(int module, unsigned int seed, int dq_line) {
int p, i, bit;
unsigned int errors;
unsigned int prv;
@ -361,6 +367,10 @@ static unsigned int sdram_write_read_check_test_pattern(int module, unsigned int
mask = MODULE_BITMASK;
#ifdef SDRAM_DELAY_PER_DQ
mask = 1 << dq_line;
#endif // SDRAM_DELAY_PER_DQ
/* Values written into CSR are Big Endian */
/* SDRAM_PHY_XDR is define 1 if SDR and 2 if DDR*/
nebo = (DFII_PIX_DATA_BYTES / SDRAM_PHY_XDR) - 1 - (module * SDRAM_PHY_DQ_DQS_RATIO)/8;
@ -403,16 +413,17 @@ static unsigned int sdram_write_read_check_test_pattern(int module, unsigned int
static int _seed_array[] = {42, 84, 36, 72, 24, 48};
static int _seed_array_length = sizeof(_seed_array) / sizeof(_seed_array[0]);
static int run_test_pattern(int module) {
static int run_test_pattern(int module, int dq_line) {
int errors = 0;
for (int i = 0; i < _seed_array_length; i++) {
errors += sdram_write_read_check_test_pattern(module, _seed_array[i]);
errors += sdram_write_read_check_test_pattern(module, _seed_array[i], dq_line);
}
return errors;
}
static void sdram_leveling_center_module(
int module, int show_short, int show_long, action_callback rst_delay, action_callback inc_delay) {
int module, int show_short, int show_long, action_callback rst_delay,
action_callback inc_delay, int dq_line) {
int i;
int show;
@ -422,14 +433,18 @@ static void sdram_leveling_center_module(
int delay_min = -1, delay_max = -1, cur_delay_min = -1;
if (show_long)
#ifdef SDRAM_DELAY_PER_DQ
printf("m%d dq_line:%d: |", module, dq_line);
#else
printf("m%d: |", module);
#endif // SDRAM_DELAY_PER_DQ
/* Find smallest working delay */
delay = 0;
working = 0;
sdram_leveling_action(module, rst_delay);
sdram_leveling_action(module, dq_line, rst_delay);
while(1) {
errors = run_test_pattern(module);
errors = run_test_pattern(module, dq_line);
last_working = working;
working = errors == 0;
show = show_long && (delay%MODULO == 0);
@ -442,14 +457,14 @@ static void sdram_leveling_center_module(
delay++;
if(delay >= SDRAM_PHY_DELAYS)
break;
sdram_leveling_action(module, inc_delay);
sdram_leveling_action(module, dq_line, inc_delay);
}
delay_max = delay_min;
cur_delay_min = delay_min;
/* Find largest working delay range */
while(1) {
errors = run_test_pattern(module);
errors = run_test_pattern(module, dq_line);
working = errors == 0;
show = show_long && (delay%MODULO == 0);
if (show)
@ -468,7 +483,7 @@ static void sdram_leveling_center_module(
delay++;
if(delay >= SDRAM_PHY_DELAYS)
break;
sdram_leveling_action(module, inc_delay);
sdram_leveling_action(module, dq_line, inc_delay);
}
if(delay_max < 0) {
delay_max = delay;
@ -494,15 +509,15 @@ static void sdram_leveling_center_module(
int retries = 8; /* Do N configs/checks and give up if failing */
while (retries > 0) {
/* Set delay. */
sdram_leveling_action(module, rst_delay);
sdram_leveling_action(module, dq_line, rst_delay);
cdelay(100);
for(i = 0; i < delay_mid; i++) {
sdram_leveling_action(module, inc_delay);
sdram_leveling_action(module, dq_line, inc_delay);
cdelay(100);
}
/* Check */
errors = run_test_pattern(module);
errors = run_test_pattern(module, dq_line);
if (errors == 0)
break;
retries--;
@ -572,7 +587,7 @@ void sdram_write_leveling_force_cmd_delay(int taps, int show) {
}
static int sdram_write_leveling_scan(int *delays, int loops, int show) {
int i, j, k;
int i, j, k, dq_line;
int err_ddrphy_wdly;
@ -591,11 +606,16 @@ static int sdram_write_leveling_scan(int *delays, int loops, int show) {
sdram_write_leveling_on();
cdelay(100);
for(i=0;i<SDRAM_PHY_MODULES;i++) {
for (dq_line = 0; dq_line < DQ_COUNT; dq_line++) {
if (show)
#ifdef SDRAM_DELAY_PER_DQ
printf(" m%d dq%d: |", i, dq_line);
#else
printf(" m%d: |", i);
#endif // SDRAM_DELAY_PER_DQ
/* Reset delay */
sdram_leveling_action(i, write_rst_delay);
sdram_leveling_action(i, dq_line, write_rst_delay);
cdelay(100);
/* Scan write delay taps */
@ -630,7 +650,7 @@ static int sdram_write_leveling_scan(int *delays, int loops, int show) {
taps_scan[j] = 0;
if (show_iter)
printf("%d", taps_scan[j]);
sdram_leveling_action(i, write_inc_delay);
sdram_leveling_action(i, dq_line, write_inc_delay);
cdelay(100);
}
if (show)
@ -662,7 +682,7 @@ static int sdram_write_leveling_scan(int *delays, int loops, int show) {
}
/* Reset delay */
sdram_leveling_action(i, write_rst_delay);
sdram_leveling_action(i, dq_line, write_rst_delay);
cdelay(100);
/* Use forced delay if configured */
@ -671,7 +691,7 @@ static int sdram_write_leveling_scan(int *delays, int loops, int show) {
/* Configure write delay */
for(j=0; j<delays[i]; j++) {
sdram_leveling_action(i, write_inc_delay);
sdram_leveling_action(i, dq_line, write_inc_delay);
cdelay(100);
}
/* Succeed only if the start of a 1s window has been found: */
@ -689,7 +709,7 @@ static int sdram_write_leveling_scan(int *delays, int loops, int show) {
/* Configure write delay */
for(j=0; j<delays[i]; j++) {
sdram_leveling_action(i, write_inc_delay);
sdram_leveling_action(i, dq_line, write_inc_delay);
cdelay(100);
}
}
@ -700,6 +720,7 @@ static int sdram_write_leveling_scan(int *delays, int loops, int show) {
printf(" delay: %02d\n", delays[i]);
}
}
}
sdram_write_leveling_off();
@ -846,7 +867,7 @@ int sdram_write_leveling(void) {
#if defined(SDRAM_PHY_WRITE_DQ_DQS_TRAINING_CAPABLE) || defined(SDRAM_PHY_WRITE_LATENCY_CALIBRATION_CAPABLE) || defined(SDRAM_PHY_READ_LEVELING_CAPABLE)
static unsigned int sdram_read_leveling_scan_module(int module, int bitslip, int show) {
static unsigned int sdram_read_leveling_scan_module(int module, int bitslip, int show, int dq_line) {
const unsigned int max_errors = _seed_array_length*READ_CHECK_TEST_PATTERN_MAX_ERRORS;
int i;
unsigned int score;
@ -856,18 +877,18 @@ static unsigned int sdram_read_leveling_scan_module(int module, int bitslip, int
score = 0;
if (show)
printf(" m%d, b%02d: |", module, bitslip);
sdram_leveling_action(module, read_rst_dq_delay);
sdram_leveling_action(module, dq_line, read_rst_dq_delay);
for(i=0;i<SDRAM_PHY_DELAYS;i++) {
int working;
int _show = (i%MODULO == 0) & show;
errors = run_test_pattern(module);
errors = run_test_pattern(module, dq_line);
working = errors == 0;
/* When any scan is working then the final score will always be higher then if no scan was working */
score += (working * max_errors*SDRAM_PHY_DELAYS) + (max_errors - errors);
if (_show) {
print_scan_errors(errors);
}
sdram_leveling_action(module, read_inc_dq_delay);
sdram_leveling_action(module, dq_line, read_inc_dq_delay);
}
if (show)
printf("| ");
@ -882,20 +903,22 @@ static unsigned int sdram_read_leveling_scan_module(int module, int bitslip, int
void sdram_read_leveling(void) {
int module;
int bitslip;
int dq_line;
unsigned int score;
unsigned int best_score;
int best_bitslip;
for(module=0; module<SDRAM_PHY_MODULES; module++) {
for (dq_line = 0; dq_line < DQ_COUNT; dq_line++) {
/* Scan possible read windows */
best_score = 0;
best_bitslip = 0;
sdram_leveling_action(module, read_rst_dq_bitslip);
sdram_leveling_action(module, dq_line, read_rst_dq_bitslip);
for(bitslip=0; bitslip<SDRAM_PHY_BITSLIPS; bitslip++) {
/* Compute score */
score = sdram_read_leveling_scan_module(module, bitslip, 1);
score = sdram_read_leveling_scan_module(module, bitslip, 1, dq_line);
sdram_leveling_center_module(module, 1, 0,
read_rst_dq_delay, read_inc_dq_delay);
read_rst_dq_delay, read_inc_dq_delay, dq_line);
printf("\n");
if (score > best_score) {
best_bitslip = bitslip;
@ -905,21 +928,26 @@ void sdram_read_leveling(void) {
if (bitslip == SDRAM_PHY_BITSLIPS-1)
break;
/* Increment bitslip */
sdram_leveling_action(module, read_inc_dq_bitslip);
sdram_leveling_action(module, dq_line, read_inc_dq_bitslip);
}
/* Select best read window */
#ifdef SDRAM_DELAY_PER_DQ
printf(" best: m%d, b%02d, dq_line%d ", module, best_bitslip, dq_line);
#else
printf(" best: m%d, b%02d ", module, best_bitslip);
sdram_leveling_action(module, read_rst_dq_bitslip);
#endif // SDRAM_DELAY_PER_DQ
sdram_leveling_action(module, dq_line, read_rst_dq_bitslip);
for (bitslip=0; bitslip<best_bitslip; bitslip++)
sdram_leveling_action(module, read_inc_dq_bitslip);
sdram_leveling_action(module, dq_line, read_inc_dq_bitslip);
/* Re-do leveling on best read window*/
sdram_leveling_center_module(module, 1, 0,
read_rst_dq_delay, read_inc_dq_delay);
read_rst_dq_delay, read_inc_dq_delay, dq_line);
printf("\n");
}
}
}
#endif // SDRAM_PHY_READ_LEVELING_CAPABLE
@ -935,12 +963,14 @@ static void sdram_write_latency_calibration(void) {
int i;
int module;
int bitslip;
int dq_line;
unsigned int score;
unsigned int subscore;
unsigned int best_score;
int best_bitslip;
for(module = 0; module < SDRAM_PHY_MODULES; module++) {
for (dq_line = 0; dq_line < DQ_COUNT; dq_line++) {
/* Scan possible write windows */
best_score = 0;
best_bitslip = -1;
@ -948,25 +978,24 @@ static void sdram_write_latency_calibration(void) {
if (SDRAM_WLC_DEBUG)
printf("m%d wb%02d:\n", module, bitslip);
/* Reset bitslip */
sdram_leveling_action(module, write_rst_dq_bitslip);
sdram_leveling_action(module, dq_line, write_rst_dq_bitslip);
for (i=0; i<bitslip; i++) {
sdram_leveling_action(module, write_inc_dq_bitslip);
sdram_leveling_action(module, dq_line, write_inc_dq_bitslip);
}
score = 0;
sdram_leveling_action(module, read_rst_dq_bitslip);
sdram_leveling_action(module, dq_line, read_rst_dq_bitslip);
for(i=0; i<SDRAM_PHY_BITSLIPS; i++) {
/* Compute score */
const int debug = SDRAM_WLC_DEBUG; // Local variable should be optimized out
subscore = sdram_read_leveling_scan_module(module, i, debug);
subscore = sdram_read_leveling_scan_module(module, i, debug, dq_line);
// If SDRAM_WRITE_LATENCY_CALIBRATION_DEBUG was not defined, SDRAM_WLC_DEBUG will be defined as 0, so if(0) should be optimized out
if (debug)
printf("\n");
score = subscore > score ? subscore : score;
/* Increment bitslip */
sdram_leveling_action(module, read_inc_dq_bitslip);
sdram_leveling_action(module, dq_line, read_inc_dq_bitslip);
}
if (score > best_score) {
best_bitslip = bitslip;
@ -985,19 +1014,24 @@ static void sdram_write_latency_calibration(void) {
if (bitslip == -1)
printf("m%d:- ", module);
else
#ifdef SDRAM_DELAY_PER_DQ
printf("m%d dq%d:%d ", module, dq_line, bitslip);
#else
printf("m%d:%d ", module, bitslip);
#endif // SDRAM_DELAY_PER_DQ
if (SDRAM_WLC_DEBUG)
printf("\n");
/* Reset bitslip */
sdram_leveling_action(module, write_rst_dq_bitslip);
sdram_leveling_action(module, dq_line, write_rst_dq_bitslip);
for (i=0; i<bitslip; i++) {
sdram_leveling_action(module, write_inc_dq_bitslip);
sdram_leveling_action(module, dq_line, write_inc_dq_bitslip);
}
}
printf("\n");
}
}
#endif // SDRAM_PHY_WRITE_LATENCY_CALIBRATION_CAPABLE
@ -1007,43 +1041,46 @@ static void sdram_write_latency_calibration(void) {
#ifdef SDRAM_PHY_WRITE_DQ_DQS_TRAINING_CAPABLE
static void sdram_read_leveling_best_bitslip(int module) {
static void sdram_read_leveling_best_bitslip(int module, int dq_line) {
unsigned int score;
int bitslip;
int best_bitslip = 0;
unsigned int best_score = 0;
sdram_leveling_action(module, read_rst_dq_bitslip);
sdram_leveling_action(module, dq_line, read_rst_dq_bitslip);
for(bitslip=0; bitslip<SDRAM_PHY_BITSLIPS; bitslip++) {
score = sdram_read_leveling_scan_module(module, bitslip, 0);
score = sdram_read_leveling_scan_module(module, bitslip, 0, dq_line);
sdram_leveling_center_module(module, 0, 0,
read_rst_dq_delay, read_inc_dq_delay);
read_rst_dq_delay, read_inc_dq_delay, dq_line);
if (score > best_score) {
best_bitslip = bitslip;
best_score = score;
}
if (bitslip == SDRAM_PHY_BITSLIPS-1)
break;
sdram_leveling_action(module, read_inc_dq_bitslip);
sdram_leveling_action(module, dq_line, read_inc_dq_bitslip);
}
/* Select best read window and re-center it */
sdram_leveling_action(module, read_rst_dq_bitslip);
sdram_leveling_action(module, dq_line, read_rst_dq_bitslip);
for (bitslip=0; bitslip<best_bitslip; bitslip++)
sdram_leveling_action(module, read_inc_dq_bitslip);
sdram_leveling_action(module, dq_line, read_inc_dq_bitslip);
sdram_leveling_center_module(module, 0, 0,
read_rst_dq_delay, read_inc_dq_delay);
read_rst_dq_delay, read_inc_dq_delay, dq_line);
}
static void sdram_write_dq_dqs_training(void) {
int module;
int dq_line;
for(module=0; module<SDRAM_PHY_MODULES; module++) {
for (dq_line = 0; dq_line < DQ_COUNT; dq_line++) {
/* Find best bitslip */
sdram_read_leveling_best_bitslip(module);
sdram_read_leveling_best_bitslip(module, dq_line);
/* Center DQ-DQS window */
sdram_leveling_center_module(module, 1, 1,
write_rst_dq_delay, write_inc_dq_delay);
write_rst_dq_delay, write_inc_dq_delay, dq_line);
}
}
}
@ -1055,23 +1092,26 @@ static void sdram_write_dq_dqs_training(void) {
int sdram_leveling(void) {
int module;
int dq_line;
sdram_software_control_on();
for(module=0; module<SDRAM_PHY_MODULES; module++) {
for (dq_line = 0; dq_line < DQ_COUNT; dq_line++) {
#ifdef SDRAM_PHY_WRITE_LEVELING_CAPABLE
sdram_leveling_action(module, write_rst_delay);
sdram_leveling_action(module, dq_line, write_rst_delay);
#ifdef SDRAM_PHY_BITSLIPS
sdram_leveling_action(module, write_rst_dq_bitslip);
sdram_leveling_action(module, dq_line, write_rst_dq_bitslip);
#endif // SDRAM_PHY_BITSLIPS
#endif // SDRAM_PHY_WRITE_LEVELING_CAPABLE
#ifdef SDRAM_PHY_READ_LEVELING_CAPABLE
sdram_leveling_action(module, read_rst_dq_delay);
sdram_leveling_action(module, dq_line, read_rst_dq_delay);
#ifdef SDRAM_PHY_BITSLIPS
sdram_leveling_action(module, read_rst_dq_bitslip);
sdram_leveling_action(module, dq_line, read_rst_dq_bitslip);
#endif // SDRAM_PHY_BITSLIPS
#endif // SDRAM_PHY_READ_LEVELING_CAPABLE
}
}
#ifdef SDRAM_PHY_WRITE_LEVELING_CAPABLE
printf("Write leveling:\n");