diff mbox series

[5/5] powerpc/pci: Add config option for using all 256 PCI buses

Message ID 20220706104308.5390-6-pali@kernel.org
State New
Headers show
Series powerpc/pci: Cleanup unused code and enable 256 PCI buses | expand

Commit Message

Pali Rohár July 6, 2022, 10:43 a.m. UTC 
By default on PPC32 are PCI bus numbers unique across all PCI domains.
So system could have only 256 PCI buses independently of available
PCI domains.

This is due to filling DT property pci-OF-bus-map which does not reflect
multi-domain setup.

On all powerpc platforms except chrp and powermac there is no DT property
pci-OF-bus-map anymore and therefore it is possible on non-chrp/powermac
platforms to avoid this limitation of maximal number of 256 PCI buses in
system even on multi-domain setup.

But avoiding this limitation would mean that all PCI and PCIe devices would
be present on completely different BDF addresses as every PCI domain starts
numbering PCI bueses from zero (instead of the last bus number of previous
enumerated PCI domain). Such change could break existing software which
expects fixed PCI bus numbers.

So add a new config option CONFIG_PPC_PCI_BUS_NUM_DOMAIN_DEPENDENT which
enables this change. By default it is disabled. It cause that initial value
of hose->first_busno is zero.

Signed-off-by: Pali Rohár <pali@kernel.org>
---
 arch/powerpc/Kconfig         | 11 +++++++++++
 arch/powerpc/kernel/pci_32.c |  6 ++++++
 2 files changed, 17 insertions(+)

Comments

Pali Rohár July 21, 2022, 10:21 p.m. UTC | #1 
On Wednesday 06 July 2022 12:43:08 Pali Rohár wrote:
> By default on PPC32 are PCI bus numbers unique across all PCI domains.
> So system could have only 256 PCI buses independently of available
> PCI domains.
> 
> This is due to filling DT property pci-OF-bus-map which does not reflect
> multi-domain setup.
> 
> On all powerpc platforms except chrp and powermac there is no DT property
> pci-OF-bus-map anymore and therefore it is possible on non-chrp/powermac
> platforms to avoid this limitation of maximal number of 256 PCI buses in
> system even on multi-domain setup.
> 
> But avoiding this limitation would mean that all PCI and PCIe devices would
> be present on completely different BDF addresses as every PCI domain starts
> numbering PCI bueses from zero (instead of the last bus number of previous
> enumerated PCI domain). Such change could break existing software which
> expects fixed PCI bus numbers.
> 
> So add a new config option CONFIG_PPC_PCI_BUS_NUM_DOMAIN_DEPENDENT which
> enables this change. By default it is disabled. It cause that initial value
> of hose->first_busno is zero.
> 
> Signed-off-by: Pali Rohár <pali@kernel.org>
> ---
>  arch/powerpc/Kconfig         | 11 +++++++++++
>  arch/powerpc/kernel/pci_32.c |  6 ++++++
>  2 files changed, 17 insertions(+)
> 
> diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
> index be68c1f02b79..f66084bc1dfe 100644
> --- a/arch/powerpc/Kconfig
> +++ b/arch/powerpc/Kconfig
> @@ -370,6 +370,17 @@ config PPC_DCR
>  	depends on PPC_DCR_NATIVE || PPC_DCR_MMIO
>  	default y
>  
> +config PPC_PCI_BUS_NUM_DOMAIN_DEPENDENT
> +	depends on PPC32
> +	depends on !PPC_PMAC && !PPC_CHRP
> +	bool "Assign PCI bus numbers from zero individually for each PCI domain"
> +	help
> +	  By default on PPC32 were PCI bus numbers unique across all PCI domains.
> +	  So system could have only 256 PCI buses independently of available
> +	  PCI domains. When this option is enabled then PCI bus numbers are
> +	  PCI domain dependent and each PCI controller on own domain can have
> +	  256 PCI buses, like it is on other Linux architectures.
> +

What do you think, would it be possible to set default value of this
option to enabled?

>  config PPC_OF_PLATFORM_PCI
>  	bool
>  	depends on PCI
> diff --git a/arch/powerpc/kernel/pci_32.c b/arch/powerpc/kernel/pci_32.c
> index 2f7284b68f06..433965bf37b4 100644
> --- a/arch/powerpc/kernel/pci_32.c
> +++ b/arch/powerpc/kernel/pci_32.c
> @@ -239,7 +239,9 @@ void pcibios_setup_phb_io_space(struct pci_controller *hose)
>  static int __init pcibios_init(void)
>  {
>  	struct pci_controller *hose, *tmp;
> +#ifndef CONFIG_PPC_PCI_BUS_NUM_DOMAIN_DEPENDENT
>  	int next_busno = 0;
> +#endif
>  
>  	printk(KERN_INFO "PCI: Probing PCI hardware\n");
>  
> @@ -248,13 +250,17 @@ static int __init pcibios_init(void)
>  
>  	/* Scan all of the recorded PCI controllers.  */
>  	list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
> +#ifndef CONFIG_PPC_PCI_BUS_NUM_DOMAIN_DEPENDENT
>  		if (pci_assign_all_buses)
>  			hose->first_busno = next_busno;
> +#endif
>  		hose->last_busno = 0xff;
>  		pcibios_scan_phb(hose);
>  		pci_bus_add_devices(hose->bus);
> +#ifndef CONFIG_PPC_PCI_BUS_NUM_DOMAIN_DEPENDENT
>  		if (pci_assign_all_buses || next_busno <= hose->last_busno)
>  			next_busno = hose->last_busno + pcibios_assign_bus_offset;
> +#endif
>  	}
>  
>  #if defined(CONFIG_PPC_PMAC) || defined(CONFIG_PPC_CHRP)
> -- 
> 2.20.1
>
Michael Ellerman July 26, 2022, 11:02 a.m. UTC | #2 
Pali Rohár <pali@kernel.org> writes:
> On Wednesday 06 July 2022 12:43:08 Pali Rohár wrote:
>> By default on PPC32 are PCI bus numbers unique across all PCI domains.
>> So system could have only 256 PCI buses independently of available
>> PCI domains.
>>
>> This is due to filling DT property pci-OF-bus-map which does not reflect
>> multi-domain setup.
>>
>> On all powerpc platforms except chrp and powermac there is no DT property
>> pci-OF-bus-map anymore and therefore it is possible on non-chrp/powermac
>> platforms to avoid this limitation of maximal number of 256 PCI buses in
>> system even on multi-domain setup.
>>
>> But avoiding this limitation would mean that all PCI and PCIe devices would
>> be present on completely different BDF addresses as every PCI domain starts
>> numbering PCI bueses from zero (instead of the last bus number of previous
>> enumerated PCI domain). Such change could break existing software which
>> expects fixed PCI bus numbers.
>>
>> So add a new config option CONFIG_PPC_PCI_BUS_NUM_DOMAIN_DEPENDENT which
>> enables this change. By default it is disabled. It cause that initial value
>> of hose->first_busno is zero.
>>
>> Signed-off-by: Pali Rohár <pali@kernel.org>
>> ---
>>  arch/powerpc/Kconfig         | 11 +++++++++++
>>  arch/powerpc/kernel/pci_32.c |  6 ++++++
>>  2 files changed, 17 insertions(+)
>>
>> diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
>> index be68c1f02b79..f66084bc1dfe 100644
>> --- a/arch/powerpc/Kconfig
>> +++ b/arch/powerpc/Kconfig
>> @@ -370,6 +370,17 @@ config PPC_DCR
>>  	depends on PPC_DCR_NATIVE || PPC_DCR_MMIO
>>  	default y
>>
>> +config PPC_PCI_BUS_NUM_DOMAIN_DEPENDENT
>> +	depends on PPC32
>> +	depends on !PPC_PMAC && !PPC_CHRP
>> +	bool "Assign PCI bus numbers from zero individually for each PCI domain"
>> +	help
>> +	  By default on PPC32 were PCI bus numbers unique across all PCI domains.
>> +	  So system could have only 256 PCI buses independently of available
>> +	  PCI domains. When this option is enabled then PCI bus numbers are
>> +	  PCI domain dependent and each PCI controller on own domain can have
>> +	  256 PCI buses, like it is on other Linux architectures.
>> +
>
> What do you think, would it be possible to set default value of this
> option to enabled?

My preference would be to not have the option at all, just make it the
default behaviour. Every new CONFIG option adds more combinations that
need testing, or more likely don't get well tested.

But I don't have a good feel for what could break if we turn it on, so
honestly I don't really know.

Also I do most of my 32-bit testing on pmacs, which are not affected by
the change.

So I'll probably take the series as-is, and then we can do some more
widespread testing and possibly flip the default to enabled, and then
maybe remove the option entirely in future.

cheers
Pali Rohár July 26, 2022, 11:10 a.m. UTC | #3 
On Tuesday 26 July 2022 21:02:22 Michael Ellerman wrote:
> Pali Rohár <pali@kernel.org> writes:
> > On Wednesday 06 July 2022 12:43:08 Pali Rohár wrote:
> >> By default on PPC32 are PCI bus numbers unique across all PCI domains.
> >> So system could have only 256 PCI buses independently of available
> >> PCI domains.
> >>
> >> This is due to filling DT property pci-OF-bus-map which does not reflect
> >> multi-domain setup.
> >>
> >> On all powerpc platforms except chrp and powermac there is no DT property
> >> pci-OF-bus-map anymore and therefore it is possible on non-chrp/powermac
> >> platforms to avoid this limitation of maximal number of 256 PCI buses in
> >> system even on multi-domain setup.
> >>
> >> But avoiding this limitation would mean that all PCI and PCIe devices would
> >> be present on completely different BDF addresses as every PCI domain starts
> >> numbering PCI bueses from zero (instead of the last bus number of previous
> >> enumerated PCI domain). Such change could break existing software which
> >> expects fixed PCI bus numbers.
> >>
> >> So add a new config option CONFIG_PPC_PCI_BUS_NUM_DOMAIN_DEPENDENT which
> >> enables this change. By default it is disabled. It cause that initial value
> >> of hose->first_busno is zero.
> >>
> >> Signed-off-by: Pali Rohár <pali@kernel.org>
> >> ---
> >>  arch/powerpc/Kconfig         | 11 +++++++++++
> >>  arch/powerpc/kernel/pci_32.c |  6 ++++++
> >>  2 files changed, 17 insertions(+)
> >>
> >> diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
> >> index be68c1f02b79..f66084bc1dfe 100644
> >> --- a/arch/powerpc/Kconfig
> >> +++ b/arch/powerpc/Kconfig
> >> @@ -370,6 +370,17 @@ config PPC_DCR
> >>  	depends on PPC_DCR_NATIVE || PPC_DCR_MMIO
> >>  	default y
> >>
> >> +config PPC_PCI_BUS_NUM_DOMAIN_DEPENDENT
> >> +	depends on PPC32
> >> +	depends on !PPC_PMAC && !PPC_CHRP
> >> +	bool "Assign PCI bus numbers from zero individually for each PCI domain"
> >> +	help
> >> +	  By default on PPC32 were PCI bus numbers unique across all PCI domains.
> >> +	  So system could have only 256 PCI buses independently of available
> >> +	  PCI domains. When this option is enabled then PCI bus numbers are
> >> +	  PCI domain dependent and each PCI controller on own domain can have
> >> +	  256 PCI buses, like it is on other Linux architectures.
> >> +
> >
> > What do you think, would it be possible to set default value of this
> > option to enabled?
> 
> My preference would be to not have the option at all, just make it the
> default behaviour. Every new CONFIG option adds more combinations that
> need testing, or more likely don't get well tested.
> 
> But I don't have a good feel for what could break if we turn it on, so
> honestly I don't really know.
> 
> Also I do most of my 32-bit testing on pmacs, which are not affected by
> the change.

It is because this change is incompatible with deprecated pci-OF-bus-map
which pmac uses. I do not have any pmac box for testing or development,
so I let this part as is.

If one day pci-OF-bus-map would be possible to disable on pmac then this
pci bus number change can be enabled also for pmac.

> So I'll probably take the series as-is, and then we can do some more
> widespread testing and possibly flip the default to enabled, and then
> maybe remove the option entirely in future.
> 
> cheers

I have tested it on P2020 board with 3 PCIe devices, each on own bus
where each bus is connected to different PCIe controller / domain and it
works correctly. Every PCIe device is on bus 1 bus but on different
domains.
Pali Rohár Aug. 17, 2022, 4:42 p.m. UTC | #4 
On Tuesday 26 July 2022 13:10:01 Pali Rohár wrote:
> On Tuesday 26 July 2022 21:02:22 Michael Ellerman wrote:
> > Pali Rohár <pali@kernel.org> writes:
> > > On Wednesday 06 July 2022 12:43:08 Pali Rohár wrote:
> > >> By default on PPC32 are PCI bus numbers unique across all PCI domains.
> > >> So system could have only 256 PCI buses independently of available
> > >> PCI domains.
> > >>
> > >> This is due to filling DT property pci-OF-bus-map which does not reflect
> > >> multi-domain setup.
> > >>
> > >> On all powerpc platforms except chrp and powermac there is no DT property
> > >> pci-OF-bus-map anymore and therefore it is possible on non-chrp/powermac
> > >> platforms to avoid this limitation of maximal number of 256 PCI buses in
> > >> system even on multi-domain setup.
> > >>
> > >> But avoiding this limitation would mean that all PCI and PCIe devices would
> > >> be present on completely different BDF addresses as every PCI domain starts
> > >> numbering PCI bueses from zero (instead of the last bus number of previous
> > >> enumerated PCI domain). Such change could break existing software which
> > >> expects fixed PCI bus numbers.
> > >>
> > >> So add a new config option CONFIG_PPC_PCI_BUS_NUM_DOMAIN_DEPENDENT which
> > >> enables this change. By default it is disabled. It cause that initial value
> > >> of hose->first_busno is zero.
> > >>
> > >> Signed-off-by: Pali Rohár <pali@kernel.org>
> > >> ---
> > >>  arch/powerpc/Kconfig         | 11 +++++++++++
> > >>  arch/powerpc/kernel/pci_32.c |  6 ++++++
> > >>  2 files changed, 17 insertions(+)
> > >>
> > >> diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
> > >> index be68c1f02b79..f66084bc1dfe 100644
> > >> --- a/arch/powerpc/Kconfig
> > >> +++ b/arch/powerpc/Kconfig
> > >> @@ -370,6 +370,17 @@ config PPC_DCR
> > >>  	depends on PPC_DCR_NATIVE || PPC_DCR_MMIO
> > >>  	default y
> > >>
> > >> +config PPC_PCI_BUS_NUM_DOMAIN_DEPENDENT
> > >> +	depends on PPC32
> > >> +	depends on !PPC_PMAC && !PPC_CHRP
> > >> +	bool "Assign PCI bus numbers from zero individually for each PCI domain"
> > >> +	help
> > >> +	  By default on PPC32 were PCI bus numbers unique across all PCI domains.
> > >> +	  So system could have only 256 PCI buses independently of available
> > >> +	  PCI domains. When this option is enabled then PCI bus numbers are
> > >> +	  PCI domain dependent and each PCI controller on own domain can have
> > >> +	  256 PCI buses, like it is on other Linux architectures.
> > >> +
> > >
> > > What do you think, would it be possible to set default value of this
> > > option to enabled?
> > 
> > My preference would be to not have the option at all, just make it the
> > default behaviour. Every new CONFIG option adds more combinations that
> > need testing, or more likely don't get well tested.
> > 
> > But I don't have a good feel for what could break if we turn it on, so
> > honestly I don't really know.
> > 
> > Also I do most of my 32-bit testing on pmacs, which are not affected by
> > the change.
> 
> It is because this change is incompatible with deprecated pci-OF-bus-map
> which pmac uses. I do not have any pmac box for testing or development,
> so I let this part as is.
> 
> If one day pci-OF-bus-map would be possible to disable on pmac then this
> pci bus number change can be enabled also for pmac.

Hello! I have created this patch which allows to disable deprecated
pci-OF-bus-map on powermac and allow to enable this new config option
PPC_PCI_BUS_NUM_DOMAIN_DEPENDENT also on powermac.

So you can test this option too on your powermac boxes.

I'm really not sure if that pci-OF-bus-map is required and for which
platforms or software...

Patch for allowing to disable pci-OF-bus-map is here:
https://patchwork.ozlabs.org/project/linuxppc-dev/patch/20220817163927.24453-1-pali@kernel.org/

> > So I'll probably take the series as-is, and then we can do some more
> > widespread testing and possibly flip the default to enabled, and then
> > maybe remove the option entirely in future.
> > 
> > cheers
> 
> I have tested it on P2020 board with 3 PCIe devices, each on own bus
> where each bus is connected to different PCIe controller / domain and it
> works correctly. Every PCIe device is on bus 1 bus but on different
> domains.
diff mbox series

Patch

diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
index be68c1f02b79..f66084bc1dfe 100644
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -370,6 +370,17 @@  config PPC_DCR
 	depends on PPC_DCR_NATIVE || PPC_DCR_MMIO
 	default y
 
+config PPC_PCI_BUS_NUM_DOMAIN_DEPENDENT
+	depends on PPC32
+	depends on !PPC_PMAC && !PPC_CHRP
+	bool "Assign PCI bus numbers from zero individually for each PCI domain"
+	help
+	  By default on PPC32 were PCI bus numbers unique across all PCI domains.
+	  So system could have only 256 PCI buses independently of available
+	  PCI domains. When this option is enabled then PCI bus numbers are
+	  PCI domain dependent and each PCI controller on own domain can have
+	  256 PCI buses, like it is on other Linux architectures.
+
 config PPC_OF_PLATFORM_PCI
 	bool
 	depends on PCI
diff --git a/arch/powerpc/kernel/pci_32.c b/arch/powerpc/kernel/pci_32.c
index 2f7284b68f06..433965bf37b4 100644
--- a/arch/powerpc/kernel/pci_32.c
+++ b/arch/powerpc/kernel/pci_32.c
@@ -239,7 +239,9 @@  void pcibios_setup_phb_io_space(struct pci_controller *hose)
 static int __init pcibios_init(void)
 {
 	struct pci_controller *hose, *tmp;
+#ifndef CONFIG_PPC_PCI_BUS_NUM_DOMAIN_DEPENDENT
 	int next_busno = 0;
+#endif
 
 	printk(KERN_INFO "PCI: Probing PCI hardware\n");
 
@@ -248,13 +250,17 @@  static int __init pcibios_init(void)
 
 	/* Scan all of the recorded PCI controllers.  */
 	list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
+#ifndef CONFIG_PPC_PCI_BUS_NUM_DOMAIN_DEPENDENT
 		if (pci_assign_all_buses)
 			hose->first_busno = next_busno;
+#endif
 		hose->last_busno = 0xff;
 		pcibios_scan_phb(hose);
 		pci_bus_add_devices(hose->bus);
+#ifndef CONFIG_PPC_PCI_BUS_NUM_DOMAIN_DEPENDENT
 		if (pci_assign_all_buses || next_busno <= hose->last_busno)
 			next_busno = hose->last_busno + pcibios_assign_bus_offset;
+#endif
 	}
 
 #if defined(CONFIG_PPC_PMAC) || defined(CONFIG_PPC_CHRP)