cancel
Showing results for 
Show  only  | Search instead for 
Did you mean: 
wkulp
Observer
Observer
1,851 Views
Registered: ‎02-04-2014

Kernel panic when using ttyPS1 as default console

Jump to solution

I'm encountering a kernel panic when trying to use ttyPS1 as the default console. I'm using a custom zynqmp board and building kernel v2018.3 with the default xilinx_zynqmp_defconfig. My board has two UARTs (uart0 and uart1). However the bug still arises with just uart1, so I am omitting uart0 for now.

Here is my relevant device tree source:

chosen {
    bootargs = "clk_ignore_unused console=ttyPS1,115200n8 verbose rootwait rootfstype=nfs root=/dev/nfs rw nfsroot=(omitted),v3,tcp,rsize=4096,wsize=4096 ip=dhcp"
    stdout-path = &uart1;
}

aliases {
    serial1 = &uart1;
};

&uart1 {
    status = "okay";
    pinctrl-names = "default";
    pinctrl-0 = <&pinctrl_uart1_default>;
};


Here are the kernel boot messages when it fails:

[ 4.503374] Btrfs loaded, crc32c=crc32c-generic
[ 4.508956] ff010000.serial: ttyPS1 at MMIO 0xff010000 (irq = 22, base_baud = 6250000) is a xuartps
[ 6.173007] console [ttyPS1] enabled
[ 6.178559] rtc_zynqmp ffa60000.rtc: setting system clock to 2103-11-04 21:45:26 UTC (4223655926)
[ 6.187431] of_cfs_init
[ 6.189892] of_cfs_init: OK
[ 6.194311] pps pps0: new PPS source ptp0
[ 6.198328] macb ff0d0000.ethernet: gem-ptp-timer ptp clock registered.
[ 6.204981] IPv6: ADDRCONF(NETDEV_UP): eth0: link is not ready
[ 7.204593] macb ff0d0000.ethernet eth0: unable to generate target frequency: 125000000 Hz
[ 7.212846] macb ff0d0000.ethernet eth0: link up (1000/Full)
[ 7.218502] IPv6: ADDRCONF(NETDEV_CHANGE): eth0: link becomes ready
[ 7.236652] Sending DHCP requests ., OK
[ 7.256405] IP-Config: Got DHCP answer from (omitted), my address is (omitted)
[ 7.263798] IP-Config: Complete:
[ 7.267015] device=eth0, hwaddr=(omitted), ipaddr=(omitted), mask=255.255.0.0, gw=(omitted)
[ 7.276572] host=(omitted), domain=, nis-domain=(none)
[ 7.282220] bootserver=(omitted), rootserver=(omitted), rootpath= nameserver0=(omitted)
[ 7.291440] clk: Not disabling unused clocks
[ 7.295913] ALSA device list:
[ 7.298868] No soundcards found.
[ 7.302555] tty_init_dev: ttyPS driver does not set tty->port. This will crash the kernel later. Fix the driver!
[ 7.312762] ------------[ cut here ]------------
[ 7.317372] WARNING: CPU: 0 PID: 1 at /kernel-source//drivers/tty/tty_io.c:1323 tty_init_dev+0x18c/0x1d8
[ 7.326830] Modules linked in:
[ 7.329872] CPU: 0 PID: 1 Comm: swapper/0 Not tainted 4.14.0 #3
[ 7.335779] Hardware name: ZynqMP 3u (DT)
[ 7.339773] task: ffffffc06d846d00 task.stack: ffffff8008038000
[ 7.345676] PC is at tty_init_dev+0x18c/0x1d8
[ 7.350016] LR is at tty_init_dev+0x18c/0x1d8
[ 7.354355] pc : [<ffffff80084d5ecc>] lr : [<ffffff80084d5ecc>] pstate: 60000045
[ 7.361732] sp : ffffff800803ba80
[ 7.365031] x29: ffffff800803ba80 x28: 0000000000000000
[ 7.370326] x27: ffffff8008e13000 x26: ffffff8008acb5f8
[ 7.375620] x25: 0000000000020002 x24: ffffff8008ea2000
[ 7.380915] x23: 00000000014000c0 x22: 0000000000000000
[ 7.386210] x21: 0000000000000001 x20: ffffff8008acb5e8
[ 7.391505] x19: ffffffc06cf33000 x18: 0000000000000010
[ 7.396800] x17: 0000000000000000 x16: 0000000000000000
[ 7.402095] x15: ffffffffffffffff x14: 617263206c6c6977
[ 7.407389] x13: ffffff8008de8aa0 x12: ffffff80085018e8
[ 7.412684] x11: 2074657320746f6e x10: 0000000000000007
[ 7.417979] x9 : 7669726420535079 x8 : 6972642065687420
[ 7.423274] x7 : 786946202e726574 x6 : 0000000000000139
[ 7.428569] x5 : 0000000000000064 x4 : 0000000000000000
[ 7.433863] x3 : 0000000000000000 x2 : ffffffffffffffff
[ 7.439158] x1 : ffffff8008dfc5e8 x0 : 0000000000000064
[ 7.444453] Call trace:
[ 7.446885] Exception stack(0xffffff800803b940 to 0xffffff800803ba80)
[ 7.453309] b940: 0000000000000064 ffffff8008dfc5e8 ffffffffffffffff 0000000000000000
[ 7.461120] b960: 0000000000000000 0000000000000064 0000000000000139 786946202e726574
[ 7.468932] b980: 6972642065687420 7669726420535079 0000000000000007 2074657320746f6e
[ 7.476744] b9a0: ffffff80085018e8 ffffff8008de8aa0 617263206c6c6977 ffffffffffffffff
[ 7.484556] b9c0: 0000000000000000 0000000000000000 0000000000000010 ffffffc06cf33000
[ 7.492368] b9e0: ffffff8008acb5e8 0000000000000001 0000000000000000 00000000014000c0
[ 7.500180] ba00: ffffff8008ea2000 0000000000020002 ffffff8008acb5f8 ffffff8008e13000
[ 7.507992] ba20: 0000000000000000 ffffff800803ba80 ffffff80084d5ecc ffffff800803ba80
[ 7.515804] ba40: ffffff80084d5ecc 0000000060000045 ffffff8008c7bfe0 00000000000004e2
[ 7.523616] ba60: ffffffffffffffff 0000000000000001 ffffff800803ba80 ffffff80084d5ecc
[ 7.531428] [<ffffff80084d5ecc>] tty_init_dev+0x18c/0x1d8
[ 7.536810] [<ffffff80084d6234>] tty_open+0x31c/0x458
[ 7.541848] [<ffffff800819d850>] chrdev_open+0x98/0x1b0
[ 7.547062] [<ffffff8008194f2c>] do_dentry_open.isra.1+0x11c/0x320
[ 7.553224] [<ffffff800819605c>] vfs_open+0x44/0x78
[ 7.558086] [<ffffff80081a7ea4>] path_openat+0x534/0x11f8
[ 7.563467] [<ffffff80081a9c50>] do_filp_open+0x60/0xd8
[ 7.568674] [<ffffff80081964c0>] do_sys_open+0x170/0x210
[ 7.573969] [<ffffff800819657c>] SyS_open+0x1c/0x28
[ 7.578834] [<ffffff8008d60cf0>] kernel_init_freeable+0x164/0x1d8
[ 7.584915] [<ffffff8008a27ac8>] kernel_init+0x10/0x100
[ 7.590125] [<ffffff8008084a90>] ret_from_fork+0x10/0x18
[ 7.595425] ---[ end trace 0e1439ba15cca0be ]---
[ 7.600064] Unable to handle kernel NULL pointer dereference at virtual address 00000090
[ 7.608146] Mem abort info:
[ 7.610933] Exception class = DABT (current EL), IL = 32 bits
[ 7.616843] SET = 0, FnV = 0
[ 7.619884] EA = 0, S1PTW = 0
[ 7.623013] Data abort info:
[ 7.625886] ISV = 0, ISS = 0x00000045
[ 7.629715] CM = 0, WnR = 1
[ 7.632675] [0000000000000090] user address but active_mm is swapper
[ 7.639021] Internal error: Oops: 96000045 [#1] SMP
[ 7.643885] Modules linked in:
[ 7.646925] CPU: 0 PID: 1 Comm: swapper/0 Tainted: G W 4.14.0 #3
[ 7.654040] Hardware name: ZynqMP 3u (DT)
[ 7.658034] task: ffffffc06d846d00 task.stack: ffffff8008038000
[ 7.663937] PC is at tty_init_dev+0x84/0x1d8
[ 7.668190] LR is at tty_init_dev+0x18c/0x1d8
[ 7.672529] pc : [<ffffff80084d5dc4>] lr : [<ffffff80084d5ecc>] pstate: 60000045
[ 7.679907] sp : ffffff800803ba80
[ 7.683205] x29: ffffff800803ba80 x28: 0000000000000000
[ 7.688500] x27: ffffff8008e13000 x26: ffffff8008acb5f8
[ 7.693795] x25: 0000000000020002 x24: ffffff8008ea2000
[ 7.699089] x23: 00000000014000c0 x22: 0000000000000000
[ 7.704384] x21: 0000000000000001 x20: ffffff8008acb5e8
[ 7.709679] x19: ffffffc06cf33000 x18: 0000000000000010
[ 7.714974] x17: 0000000000000000 x16: 0000000000000000
[ 7.720269] x15: ffffffffffffffff x14: 617263206c6c6977
[ 7.725563] x13: ffffff8008de8aa0 x12: ffffff80085018e8
[ 7.730858] x11: 2074657320746f6e x10: 0000000000000007
[ 7.736153] x9 : 7669726420535079 x8 : 6972642065687420
[ 7.741448] x7 : 786946202e726574 x6 : 0000000000000139
[ 7.746743] x5 : 0000000000000064 x4 : 0000000000000000
[ 7.752037] x3 : 0000000000000000 x2 : ffffffffffffffff
[ 7.757332] x1 : ffffff8008dfc5e8 x0 : 0000000000000000
[ 7.762628] Process swapper/0 (pid: 1, stack limit = 0xffffff8008038000)
[ 7.769310] Call trace:
[ 7.771742] Exception stack(0xffffff800803b940 to 0xffffff800803ba80)
[ 7.778166] b940: 0000000000000000 ffffff8008dfc5e8 ffffffffffffffff 0000000000000000
[ 7.785978] b960: 0000000000000000 0000000000000064 0000000000000139 786946202e726574
[ 7.793790] b980: 6972642065687420 7669726420535079 0000000000000007 2074657320746f6e
[ 7.801602] b9a0: ffffff80085018e8 ffffff8008de8aa0 617263206c6c6977 ffffffffffffffff
[ 7.809414] b9c0: 0000000000000000 0000000000000000 0000000000000010 ffffffc06cf33000
[ 7.817226] b9e0: ffffff8008acb5e8 0000000000000001 0000000000000000 00000000014000c0
[ 7.825038] ba00: ffffff8008ea2000 0000000000020002 ffffff8008acb5f8 ffffff8008e13000
[ 7.832850] ba20: 0000000000000000 ffffff800803ba80 ffffff80084d5ecc ffffff800803ba80
[ 7.840662] ba40: ffffff80084d5dc4 0000000060000045 ffffff8008c7bfe0 00000000000004e2
[ 7.848474] ba60: ffffffffffffffff 0000000000000001 ffffff800803ba80 ffffff80084d5dc4
[ 7.856286] [<ffffff80084d5dc4>] tty_init_dev+0x84/0x1d8
[ 7.861581] [<ffffff80084d6234>] tty_open+0x31c/0x458
[ 7.866616] [<ffffff800819d850>] chrdev_open+0x98/0x1b0
[ 7.871823] [<ffffff8008194f2c>] do_dentry_open.isra.1+0x11c/0x320
[ 7.877986] [<ffffff800819605c>] vfs_open+0x44/0x78
[ 7.882847] [<ffffff80081a7ea4>] path_openat+0x534/0x11f8
[ 7.888229] [<ffffff80081a9c50>] do_filp_open+0x60/0xd8
[ 7.893436] [<ffffff80081964c0>] do_sys_open+0x170/0x210
[ 7.898731] [<ffffff800819657c>] SyS_open+0x1c/0x28
[ 7.903592] [<ffffff8008d60cf0>] kernel_init_freeable+0x164/0x1d8
[ 7.909668] [<ffffff8008a27ac8>] kernel_init+0x10/0x100
[ 7.914876] [<ffffff8008084a90>] ret_from_fork+0x10/0x18
[ 7.920172] Code: 2a0003f6 37f80716 f9415e60 b40003c0 (f9004813)
[ 7.926247] ---[ end trace 0e1439ba15cca0bf ]---
[ 7.930872] Kernel panic - not syncing: Attempted to kill init! exitcode=0x0000000b
[ 7.930872]
[ 7.939990] Kernel Offset: disabled
[ 7.943467] CPU features: 0x002004
[ 7.946852] Memory Limit: none
[ 7.949892] ---[ end Kernel panic - not syncing: Attempted to kill init! exitcode=0x0000000b
[ 7.949892]


Here's the weird part. If I change my device tree aliases so that I call the port serial0 instead of serial1 (even though it's still pointing toward uart1) the board boots correctly!

aliases { serial0 = &uart1; };
chosen { bootargs = "... console=ttyPS0 ..." }

[ 4.508936] ff010000.serial: ttyPS0 at MMIO 0xff010000 (irq = 22, base_baud = 6250000) is a xuartps
[ 6.180748] console [ttyPS0] enabled

I've searched the error message "tty_init_dev: ttyPS driver does not set tty->port. This will crash the kernel later. Fix the driver!" and haven't had any luck. My closest guess is that some corruption is taking place. This wiki page https://xilinx-wiki.atlassian.net/wiki/spaces/A/pages/18842340/PS+UART#PSUART-KernelConfiguration mentions a config option CONFIG_SERIAL_XILINX_NR_UARTS although I can't find it anywhere in the v2018.3 kernel. I updated the kernel driver xilinx_uartps.c by copying the latest version in linux-xlnx master, but the problem still persists.

I would greatly appreciate any help!

0 Kudos
1 Solution

Accepted Solutions
wkulp
Observer
Observer
1,789 Views
Registered: ‎02-04-2014

I resolved this issue by reverting the kernel driver xilinx_xuartps.c to the version from v2017.4

The mainline version of the driver may have a bug.

View solution in original post

4 Replies
wkulp
Observer
Observer
1,790 Views
Registered: ‎02-04-2014

I resolved this issue by reverting the kernel driver xilinx_xuartps.c to the version from v2017.4

The mainline version of the driver may have a bug.

View solution in original post

stefanct
Visitor
Visitor
1,710 Views
Registered: ‎02-05-2019

The file is actually:

drivers/tty/serial/xilinx_uartps.c

This file gets rather frequent patches so I was not willing to look into it further, but if you have to this might help:

git log --follow --stat master -- ./drivers/tty/serial/xilinx_uartps.c  | git name-rev --stdin | less

One other workaround would be to change the device tree so that the bug is not triggered but AFAICT the remapping is really the only thing that triggers it so if you want to use UART1 as anything but serial0 you are skrewed: I have tried aliasing it to serial2/PS2. This has the same effect... boots fine till about when it mounts the rootfs. So the subject is a bit misleading... it is not serial1 that is specifically bad but only serial0 works...

 

0 Kudos
nathanm2
Visitor
Visitor
1,414 Views
Registered: ‎07-19-2019

I've got a partial fix for this issue, but it has the unfortunate side-effect of reporting all consoles as "ttyPS0" consoles.

Anlysis:

This is a buffer overrun issue that seems to originate within the Xilinx UART PS driver.

The console structure created by the Xilinx UART PS driver has an index value of N, where N is taken from the 'serialN' alias within the device tree. This means a "serial1" alias would have a console index value of "1".

cdns_uart_console->index = cdns_uart_data->id;

This console index is later used within the tty_init_dev function as an offset
into the ports array of the tty_driver structure:

 

if (!tty->port)
    tty->port = driver->ports[idx];

But for Xilinx registered tty_drivers, the ports array has exactly one entry!
The only valid `idx` value is 0.

Modifying the driver to hard-code an index value of 0 would seem to be the
answer, but this reveals the next problem: the registered uart ports no longer matches the "console=ttyPS1" boot argument.

To address this problem, we can modify the console structure to use a
specialized 'match' callback.

This seems to work, but, as noted above, has a side-effect of
reporting all consoles as "ttyPS0" consoles.

[    0.730335] console [ttyPS0] enabled
[    0.733935] bootconsole [cdns0] disabled

For anyone that's interested, here's my patch:

diff --git a/drivers/tty/serial/xilinx_uartps.c b/drivers/tty/serial/xilinx_uartps.c
index 1e72f278bdfd..f594c3d8596a 100644
--- a/drivers/tty/serial/xilinx_uartps.c
+++ b/drivers/tty/serial/xilinx_uartps.c
@@ -1268,6 +1268,18 @@ static int cdns_uart_console_setup(struct console *co, char *options)

return uart_set_options(port, co, baud, parity, bits, flow);
}
+
+static int cdns_uart_console_match(struct console *co, char *name, int idx,
+ char *options)
+{
+ struct uart_driver *driver = (struct uart_driver *)co->data;
+
+ if (!strcmp(co->name, name) && driver->minor == idx)
+ return cdns_uart_console_setup(co, options);
+
+ return -ENODEV;
+}
+
#endif /* CONFIG_SERIAL_XILINX_PS_UART_CONSOLE */

#ifdef CONFIG_PM_SLEEP
@@ -1545,10 +1557,11 @@ static int cdns_uart_probe(struct platform_device *pdev)

strncpy(cdns_uart_console->name, CDNS_UART_TTY_NAME,
sizeof(cdns_uart_console->name));
- cdns_uart_console->index = cdns_uart_data->id;
+ cdns_uart_console->index = 0;
cdns_uart_console->write = cdns_uart_console_write;
cdns_uart_console->device = uart_console_device;
cdns_uart_console->setup = cdns_uart_console_setup;
+ cdns_uart_console->match = cdns_uart_console_match;
cdns_uart_console->flags = CON_PRINTBUFFER;
cdns_uart_console->data = cdns_uart_uart_driver;
cdns_uart_uart_driver->cons = cdns_uart_console;
barco2
Adventurer
Adventurer
1,068 Views
Registered: ‎02-13-2009

Seems to be fixed for the latest version of xilinx_uartps on github. However, the issue is the same for the uartlite driver and porting your patch to uartlite also gives a bootable kernel with the same side-effect of wrongly reporting as ttyUL0 instead of ttyUL1.

diff -ur kernel-source-orig/drivers/tty/serial/uartlite.c kernel-source-new/drivers/tty/serial/uartlite.c
--- kernel-source-orig/drivers/tty/serial/uartlite.c	2020-07-28 19:44:08.618802949 +0200
+++ kernel-source-new/drivers/tty/serial/uartlite.c	2020-07-28 21:58:46.614802949 +0200
@@ -534,6 +534,17 @@
 	return uart_set_options(port, co, baud, parity, bits, flow);
 }
 
+static int ulite_console_match(struct console *co, char *name, int idx,
+ char *options)
+{
+	struct uart_driver *driver = (struct uart_driver *)co->data;
+
+	if (!strcmp(co->name, name) && driver->minor == idx)
+		return ulite_console_setup(co, options);
+
+	return -ENODEV;
+}
+
 static void early_uartlite_putc(struct uart_port *port, int c)
 {
 	/*
@@ -886,10 +897,11 @@
 
 	strncpy(ulite_console->name, ULITE_NAME,
 		sizeof(ulite_console->name));
-	ulite_console->index = pdata->id;
+	ulite_console->index = 0;
 	ulite_console->write = ulite_console_write;
 	ulite_console->device = uart_console_device;
 	ulite_console->setup = ulite_console_setup;
+	ulite_console->match = ulite_console_match;
 	ulite_console->flags = CON_PRINTBUFFER;
 	ulite_uart_driver->cons = ulite_console;
 	ulite_console->data = ulite_uart_driver;