1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * PCI Bus Services, see include/linux/pci.h for further explanation.
4  *
5  * Copyright 1993 -- 1997 Drew Eckhardt, Frederic Potter,
6  * David Mosberger-Tang
7  *
8  * Copyright 1997 -- 2000 Martin Mares <mj@ucw.cz>
9  */
10 
11 #include <linux/acpi.h>
12 #include <linux/kernel.h>
13 #include <linux/delay.h>
14 #include <linux/dmi.h>
15 #include <linux/init.h>
16 #include <linux/of.h>
17 #include <linux/of_pci.h>
18 #include <linux/pci.h>
19 #include <linux/pm.h>
20 #include <linux/slab.h>
21 #include <linux/module.h>
22 #include <linux/spinlock.h>
23 #include <linux/string.h>
24 #include <linux/log2.h>
25 #include <linux/logic_pio.h>
26 #include <linux/pm_wakeup.h>
27 #include <linux/interrupt.h>
28 #include <linux/device.h>
29 #include <linux/pm_runtime.h>
30 #include <linux/pci_hotplug.h>
31 #include <linux/vmalloc.h>
32 #include <linux/pci-ats.h>
33 #include <asm/setup.h>
34 #include <asm/dma.h>
35 #include <linux/aer.h>
36 #include "pci.h"
37 
38 DEFINE_MUTEX(pci_slot_mutex);
39 
40 const char *pci_power_names[] = {
41 	"error", "D0", "D1", "D2", "D3hot", "D3cold", "unknown",
42 };
43 EXPORT_SYMBOL_GPL(pci_power_names);
44 
45 int isa_dma_bridge_buggy;
46 EXPORT_SYMBOL(isa_dma_bridge_buggy);
47 
48 int pci_pci_problems;
49 EXPORT_SYMBOL(pci_pci_problems);
50 
51 unsigned int pci_pm_d3_delay;
52 
53 static void pci_pme_list_scan(struct work_struct *work);
54 
55 static LIST_HEAD(pci_pme_list);
56 static DEFINE_MUTEX(pci_pme_list_mutex);
57 static DECLARE_DELAYED_WORK(pci_pme_work, pci_pme_list_scan);
58 
59 struct pci_pme_device {
60 	struct list_head list;
61 	struct pci_dev *dev;
62 };
63 
64 #define PME_TIMEOUT 1000 /* How long between PME checks */
65 
pci_dev_d3_sleep(struct pci_dev * dev)66 static void pci_dev_d3_sleep(struct pci_dev *dev)
67 {
68 	unsigned int delay = dev->d3_delay;
69 
70 	if (delay < pci_pm_d3_delay)
71 		delay = pci_pm_d3_delay;
72 
73 	if (delay)
74 		msleep(delay);
75 }
76 
77 #ifdef CONFIG_PCI_DOMAINS
78 int pci_domains_supported = 1;
79 #endif
80 
81 #define DEFAULT_CARDBUS_IO_SIZE		(256)
82 #define DEFAULT_CARDBUS_MEM_SIZE	(64*1024*1024)
83 /* pci=cbmemsize=nnM,cbiosize=nn can override this */
84 unsigned long pci_cardbus_io_size = DEFAULT_CARDBUS_IO_SIZE;
85 unsigned long pci_cardbus_mem_size = DEFAULT_CARDBUS_MEM_SIZE;
86 
87 #define DEFAULT_HOTPLUG_IO_SIZE		(256)
88 #define DEFAULT_HOTPLUG_MEM_SIZE	(2*1024*1024)
89 /* pci=hpmemsize=nnM,hpiosize=nn can override this */
90 unsigned long pci_hotplug_io_size  = DEFAULT_HOTPLUG_IO_SIZE;
91 unsigned long pci_hotplug_mem_size = DEFAULT_HOTPLUG_MEM_SIZE;
92 
93 #define DEFAULT_HOTPLUG_BUS_SIZE	1
94 unsigned long pci_hotplug_bus_size = DEFAULT_HOTPLUG_BUS_SIZE;
95 
96 enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_DEFAULT;
97 
98 /*
99  * The default CLS is used if arch didn't set CLS explicitly and not
100  * all pci devices agree on the same value.  Arch can override either
101  * the dfl or actual value as it sees fit.  Don't forget this is
102  * measured in 32-bit words, not bytes.
103  */
104 u8 pci_dfl_cache_line_size = L1_CACHE_BYTES >> 2;
105 u8 pci_cache_line_size;
106 
107 /*
108  * If we set up a device for bus mastering, we need to check the latency
109  * timer as certain BIOSes forget to set it properly.
110  */
111 unsigned int pcibios_max_latency = 255;
112 
113 /* If set, the PCIe ARI capability will not be used. */
114 static bool pcie_ari_disabled;
115 
116 /* If set, the PCIe ATS capability will not be used. */
117 static bool pcie_ats_disabled;
118 
119 /* If set, the PCI config space of each device is printed during boot. */
120 bool pci_early_dump;
121 
pci_ats_disabled(void)122 bool pci_ats_disabled(void)
123 {
124 	return pcie_ats_disabled;
125 }
126 
127 /* Disable bridge_d3 for all PCIe ports */
128 static bool pci_bridge_d3_disable;
129 /* Force bridge_d3 for all PCIe ports */
130 static bool pci_bridge_d3_force;
131 
pcie_port_pm_setup(char * str)132 static int __init pcie_port_pm_setup(char *str)
133 {
134 	if (!strcmp(str, "off"))
135 		pci_bridge_d3_disable = true;
136 	else if (!strcmp(str, "force"))
137 		pci_bridge_d3_force = true;
138 	return 1;
139 }
140 __setup("pcie_port_pm=", pcie_port_pm_setup);
141 
142 /* Time to wait after a reset for device to become responsive */
143 #define PCIE_RESET_READY_POLL_MS 60000
144 
145 /**
146  * pci_bus_max_busnr - returns maximum PCI bus number of given bus' children
147  * @bus: pointer to PCI bus structure to search
148  *
149  * Given a PCI bus, returns the highest PCI bus number present in the set
150  * including the given PCI bus and its list of child PCI buses.
151  */
pci_bus_max_busnr(struct pci_bus * bus)152 unsigned char pci_bus_max_busnr(struct pci_bus *bus)
153 {
154 	struct pci_bus *tmp;
155 	unsigned char max, n;
156 
157 	max = bus->busn_res.end;
158 	list_for_each_entry(tmp, &bus->children, node) {
159 		n = pci_bus_max_busnr(tmp);
160 		if (n > max)
161 			max = n;
162 	}
163 	return max;
164 }
165 EXPORT_SYMBOL_GPL(pci_bus_max_busnr);
166 
167 #ifdef CONFIG_HAS_IOMEM
pci_ioremap_bar(struct pci_dev * pdev,int bar)168 void __iomem *pci_ioremap_bar(struct pci_dev *pdev, int bar)
169 {
170 	struct resource *res = &pdev->resource[bar];
171 
172 	/*
173 	 * Make sure the BAR is actually a memory resource, not an IO resource
174 	 */
175 	if (res->flags & IORESOURCE_UNSET || !(res->flags & IORESOURCE_MEM)) {
176 		pci_warn(pdev, "can't ioremap BAR %d: %pR\n", bar, res);
177 		return NULL;
178 	}
179 	return ioremap_nocache(res->start, resource_size(res));
180 }
181 EXPORT_SYMBOL_GPL(pci_ioremap_bar);
182 
pci_ioremap_wc_bar(struct pci_dev * pdev,int bar)183 void __iomem *pci_ioremap_wc_bar(struct pci_dev *pdev, int bar)
184 {
185 	/*
186 	 * Make sure the BAR is actually a memory resource, not an IO resource
187 	 */
188 	if (!(pci_resource_flags(pdev, bar) & IORESOURCE_MEM)) {
189 		WARN_ON(1);
190 		return NULL;
191 	}
192 	return ioremap_wc(pci_resource_start(pdev, bar),
193 			  pci_resource_len(pdev, bar));
194 }
195 EXPORT_SYMBOL_GPL(pci_ioremap_wc_bar);
196 #endif
197 
198 /**
199  * pci_dev_str_match_path - test if a path string matches a device
200  * @dev:    the PCI device to test
201  * @p:      string to match the device against
202  * @endptr: pointer to the string after the match
203  *
204  * Test if a string (typically from a kernel parameter) formatted as a
205  * path of device/function addresses matches a PCI device. The string must
206  * be of the form:
207  *
208  *   [<domain>:]<bus>:<device>.<func>[/<device>.<func>]*
209  *
210  * A path for a device can be obtained using 'lspci -t'.  Using a path
211  * is more robust against bus renumbering than using only a single bus,
212  * device and function address.
213  *
214  * Returns 1 if the string matches the device, 0 if it does not and
215  * a negative error code if it fails to parse the string.
216  */
pci_dev_str_match_path(struct pci_dev * dev,const char * path,const char ** endptr)217 static int pci_dev_str_match_path(struct pci_dev *dev, const char *path,
218 				  const char **endptr)
219 {
220 	int ret;
221 	int seg, bus, slot, func;
222 	char *wpath, *p;
223 	char end;
224 
225 	*endptr = strchrnul(path, ';');
226 
227 	wpath = kmemdup_nul(path, *endptr - path, GFP_ATOMIC);
228 	if (!wpath)
229 		return -ENOMEM;
230 
231 	while (1) {
232 		p = strrchr(wpath, '/');
233 		if (!p)
234 			break;
235 		ret = sscanf(p, "/%x.%x%c", &slot, &func, &end);
236 		if (ret != 2) {
237 			ret = -EINVAL;
238 			goto free_and_exit;
239 		}
240 
241 		if (dev->devfn != PCI_DEVFN(slot, func)) {
242 			ret = 0;
243 			goto free_and_exit;
244 		}
245 
246 		/*
247 		 * Note: we don't need to get a reference to the upstream
248 		 * bridge because we hold a reference to the top level
249 		 * device which should hold a reference to the bridge,
250 		 * and so on.
251 		 */
252 		dev = pci_upstream_bridge(dev);
253 		if (!dev) {
254 			ret = 0;
255 			goto free_and_exit;
256 		}
257 
258 		*p = 0;
259 	}
260 
261 	ret = sscanf(wpath, "%x:%x:%x.%x%c", &seg, &bus, &slot,
262 		     &func, &end);
263 	if (ret != 4) {
264 		seg = 0;
265 		ret = sscanf(wpath, "%x:%x.%x%c", &bus, &slot, &func, &end);
266 		if (ret != 3) {
267 			ret = -EINVAL;
268 			goto free_and_exit;
269 		}
270 	}
271 
272 	ret = (seg == pci_domain_nr(dev->bus) &&
273 	       bus == dev->bus->number &&
274 	       dev->devfn == PCI_DEVFN(slot, func));
275 
276 free_and_exit:
277 	kfree(wpath);
278 	return ret;
279 }
280 
281 /**
282  * pci_dev_str_match - test if a string matches a device
283  * @dev:    the PCI device to test
284  * @p:      string to match the device against
285  * @endptr: pointer to the string after the match
286  *
287  * Test if a string (typically from a kernel parameter) matches a specified
288  * PCI device. The string may be of one of the following formats:
289  *
290  *   [<domain>:]<bus>:<device>.<func>[/<device>.<func>]*
291  *   pci:<vendor>:<device>[:<subvendor>:<subdevice>]
292  *
293  * The first format specifies a PCI bus/device/function address which
294  * may change if new hardware is inserted, if motherboard firmware changes,
295  * or due to changes caused in kernel parameters. If the domain is
296  * left unspecified, it is taken to be 0.  In order to be robust against
297  * bus renumbering issues, a path of PCI device/function numbers may be used
298  * to address the specific device.  The path for a device can be determined
299  * through the use of 'lspci -t'.
300  *
301  * The second format matches devices using IDs in the configuration
302  * space which may match multiple devices in the system. A value of 0
303  * for any field will match all devices. (Note: this differs from
304  * in-kernel code that uses PCI_ANY_ID which is ~0; this is for
305  * legacy reasons and convenience so users don't have to specify
306  * FFFFFFFFs on the command line.)
307  *
308  * Returns 1 if the string matches the device, 0 if it does not and
309  * a negative error code if the string cannot be parsed.
310  */
pci_dev_str_match(struct pci_dev * dev,const char * p,const char ** endptr)311 static int pci_dev_str_match(struct pci_dev *dev, const char *p,
312 			     const char **endptr)
313 {
314 	int ret;
315 	int count;
316 	unsigned short vendor, device, subsystem_vendor, subsystem_device;
317 
318 	if (strncmp(p, "pci:", 4) == 0) {
319 		/* PCI vendor/device (subvendor/subdevice) IDs are specified */
320 		p += 4;
321 		ret = sscanf(p, "%hx:%hx:%hx:%hx%n", &vendor, &device,
322 			     &subsystem_vendor, &subsystem_device, &count);
323 		if (ret != 4) {
324 			ret = sscanf(p, "%hx:%hx%n", &vendor, &device, &count);
325 			if (ret != 2)
326 				return -EINVAL;
327 
328 			subsystem_vendor = 0;
329 			subsystem_device = 0;
330 		}
331 
332 		p += count;
333 
334 		if ((!vendor || vendor == dev->vendor) &&
335 		    (!device || device == dev->device) &&
336 		    (!subsystem_vendor ||
337 			    subsystem_vendor == dev->subsystem_vendor) &&
338 		    (!subsystem_device ||
339 			    subsystem_device == dev->subsystem_device))
340 			goto found;
341 	} else {
342 		/*
343 		 * PCI Bus, Device, Function IDs are specified
344 		 *  (optionally, may include a path of devfns following it)
345 		 */
346 		ret = pci_dev_str_match_path(dev, p, &p);
347 		if (ret < 0)
348 			return ret;
349 		else if (ret)
350 			goto found;
351 	}
352 
353 	*endptr = p;
354 	return 0;
355 
356 found:
357 	*endptr = p;
358 	return 1;
359 }
360 
__pci_find_next_cap_ttl(struct pci_bus * bus,unsigned int devfn,u8 pos,int cap,int * ttl)361 static int __pci_find_next_cap_ttl(struct pci_bus *bus, unsigned int devfn,
362 				   u8 pos, int cap, int *ttl)
363 {
364 	u8 id;
365 	u16 ent;
366 
367 	pci_bus_read_config_byte(bus, devfn, pos, &pos);
368 
369 	while ((*ttl)--) {
370 		if (pos < 0x40)
371 			break;
372 		pos &= ~3;
373 		pci_bus_read_config_word(bus, devfn, pos, &ent);
374 
375 		id = ent & 0xff;
376 		if (id == 0xff)
377 			break;
378 		if (id == cap)
379 			return pos;
380 		pos = (ent >> 8);
381 	}
382 	return 0;
383 }
384 
__pci_find_next_cap(struct pci_bus * bus,unsigned int devfn,u8 pos,int cap)385 static int __pci_find_next_cap(struct pci_bus *bus, unsigned int devfn,
386 			       u8 pos, int cap)
387 {
388 	int ttl = PCI_FIND_CAP_TTL;
389 
390 	return __pci_find_next_cap_ttl(bus, devfn, pos, cap, &ttl);
391 }
392 
pci_find_next_capability(struct pci_dev * dev,u8 pos,int cap)393 int pci_find_next_capability(struct pci_dev *dev, u8 pos, int cap)
394 {
395 	return __pci_find_next_cap(dev->bus, dev->devfn,
396 				   pos + PCI_CAP_LIST_NEXT, cap);
397 }
398 EXPORT_SYMBOL_GPL(pci_find_next_capability);
399 
__pci_bus_find_cap_start(struct pci_bus * bus,unsigned int devfn,u8 hdr_type)400 static int __pci_bus_find_cap_start(struct pci_bus *bus,
401 				    unsigned int devfn, u8 hdr_type)
402 {
403 	u16 status;
404 
405 	pci_bus_read_config_word(bus, devfn, PCI_STATUS, &status);
406 	if (!(status & PCI_STATUS_CAP_LIST))
407 		return 0;
408 
409 	switch (hdr_type) {
410 	case PCI_HEADER_TYPE_NORMAL:
411 	case PCI_HEADER_TYPE_BRIDGE:
412 		return PCI_CAPABILITY_LIST;
413 	case PCI_HEADER_TYPE_CARDBUS:
414 		return PCI_CB_CAPABILITY_LIST;
415 	}
416 
417 	return 0;
418 }
419 
420 /**
421  * pci_find_capability - query for devices' capabilities
422  * @dev: PCI device to query
423  * @cap: capability code
424  *
425  * Tell if a device supports a given PCI capability.
426  * Returns the address of the requested capability structure within the
427  * device's PCI configuration space or 0 in case the device does not
428  * support it.  Possible values for @cap:
429  *
430  *  %PCI_CAP_ID_PM           Power Management
431  *  %PCI_CAP_ID_AGP          Accelerated Graphics Port
432  *  %PCI_CAP_ID_VPD          Vital Product Data
433  *  %PCI_CAP_ID_SLOTID       Slot Identification
434  *  %PCI_CAP_ID_MSI          Message Signalled Interrupts
435  *  %PCI_CAP_ID_CHSWP        CompactPCI HotSwap
436  *  %PCI_CAP_ID_PCIX         PCI-X
437  *  %PCI_CAP_ID_EXP          PCI Express
438  */
pci_find_capability(struct pci_dev * dev,int cap)439 int pci_find_capability(struct pci_dev *dev, int cap)
440 {
441 	int pos;
442 
443 	pos = __pci_bus_find_cap_start(dev->bus, dev->devfn, dev->hdr_type);
444 	if (pos)
445 		pos = __pci_find_next_cap(dev->bus, dev->devfn, pos, cap);
446 
447 	return pos;
448 }
449 EXPORT_SYMBOL(pci_find_capability);
450 
451 /**
452  * pci_bus_find_capability - query for devices' capabilities
453  * @bus:   the PCI bus to query
454  * @devfn: PCI device to query
455  * @cap:   capability code
456  *
457  * Like pci_find_capability() but works for pci devices that do not have a
458  * pci_dev structure set up yet.
459  *
460  * Returns the address of the requested capability structure within the
461  * device's PCI configuration space or 0 in case the device does not
462  * support it.
463  */
pci_bus_find_capability(struct pci_bus * bus,unsigned int devfn,int cap)464 int pci_bus_find_capability(struct pci_bus *bus, unsigned int devfn, int cap)
465 {
466 	int pos;
467 	u8 hdr_type;
468 
469 	pci_bus_read_config_byte(bus, devfn, PCI_HEADER_TYPE, &hdr_type);
470 
471 	pos = __pci_bus_find_cap_start(bus, devfn, hdr_type & 0x7f);
472 	if (pos)
473 		pos = __pci_find_next_cap(bus, devfn, pos, cap);
474 
475 	return pos;
476 }
477 EXPORT_SYMBOL(pci_bus_find_capability);
478 
479 /**
480  * pci_find_next_ext_capability - Find an extended capability
481  * @dev: PCI device to query
482  * @start: address at which to start looking (0 to start at beginning of list)
483  * @cap: capability code
484  *
485  * Returns the address of the next matching extended capability structure
486  * within the device's PCI configuration space or 0 if the device does
487  * not support it.  Some capabilities can occur several times, e.g., the
488  * vendor-specific capability, and this provides a way to find them all.
489  */
pci_find_next_ext_capability(struct pci_dev * dev,int start,int cap)490 int pci_find_next_ext_capability(struct pci_dev *dev, int start, int cap)
491 {
492 	u32 header;
493 	int ttl;
494 	int pos = PCI_CFG_SPACE_SIZE;
495 
496 	/* minimum 8 bytes per capability */
497 	ttl = (PCI_CFG_SPACE_EXP_SIZE - PCI_CFG_SPACE_SIZE) / 8;
498 
499 	if (dev->cfg_size <= PCI_CFG_SPACE_SIZE)
500 		return 0;
501 
502 	if (start)
503 		pos = start;
504 
505 	if (pci_read_config_dword(dev, pos, &header) != PCIBIOS_SUCCESSFUL)
506 		return 0;
507 
508 	/*
509 	 * If we have no capabilities, this is indicated by cap ID,
510 	 * cap version and next pointer all being 0.
511 	 */
512 	if (header == 0)
513 		return 0;
514 
515 	while (ttl-- > 0) {
516 		if (PCI_EXT_CAP_ID(header) == cap && pos != start)
517 			return pos;
518 
519 		pos = PCI_EXT_CAP_NEXT(header);
520 		if (pos < PCI_CFG_SPACE_SIZE)
521 			break;
522 
523 		if (pci_read_config_dword(dev, pos, &header) != PCIBIOS_SUCCESSFUL)
524 			break;
525 	}
526 
527 	return 0;
528 }
529 EXPORT_SYMBOL_GPL(pci_find_next_ext_capability);
530 
531 /**
532  * pci_find_ext_capability - Find an extended capability
533  * @dev: PCI device to query
534  * @cap: capability code
535  *
536  * Returns the address of the requested extended capability structure
537  * within the device's PCI configuration space or 0 if the device does
538  * not support it.  Possible values for @cap:
539  *
540  *  %PCI_EXT_CAP_ID_ERR		Advanced Error Reporting
541  *  %PCI_EXT_CAP_ID_VC		Virtual Channel
542  *  %PCI_EXT_CAP_ID_DSN		Device Serial Number
543  *  %PCI_EXT_CAP_ID_PWR		Power Budgeting
544  */
pci_find_ext_capability(struct pci_dev * dev,int cap)545 int pci_find_ext_capability(struct pci_dev *dev, int cap)
546 {
547 	return pci_find_next_ext_capability(dev, 0, cap);
548 }
549 EXPORT_SYMBOL_GPL(pci_find_ext_capability);
550 
__pci_find_next_ht_cap(struct pci_dev * dev,int pos,int ht_cap)551 static int __pci_find_next_ht_cap(struct pci_dev *dev, int pos, int ht_cap)
552 {
553 	int rc, ttl = PCI_FIND_CAP_TTL;
554 	u8 cap, mask;
555 
556 	if (ht_cap == HT_CAPTYPE_SLAVE || ht_cap == HT_CAPTYPE_HOST)
557 		mask = HT_3BIT_CAP_MASK;
558 	else
559 		mask = HT_5BIT_CAP_MASK;
560 
561 	pos = __pci_find_next_cap_ttl(dev->bus, dev->devfn, pos,
562 				      PCI_CAP_ID_HT, &ttl);
563 	while (pos) {
564 		rc = pci_read_config_byte(dev, pos + 3, &cap);
565 		if (rc != PCIBIOS_SUCCESSFUL)
566 			return 0;
567 
568 		if ((cap & mask) == ht_cap)
569 			return pos;
570 
571 		pos = __pci_find_next_cap_ttl(dev->bus, dev->devfn,
572 					      pos + PCI_CAP_LIST_NEXT,
573 					      PCI_CAP_ID_HT, &ttl);
574 	}
575 
576 	return 0;
577 }
578 /**
579  * pci_find_next_ht_capability - query a device's Hypertransport capabilities
580  * @dev: PCI device to query
581  * @pos: Position from which to continue searching
582  * @ht_cap: Hypertransport capability code
583  *
584  * To be used in conjunction with pci_find_ht_capability() to search for
585  * all capabilities matching @ht_cap. @pos should always be a value returned
586  * from pci_find_ht_capability().
587  *
588  * NB. To be 100% safe against broken PCI devices, the caller should take
589  * steps to avoid an infinite loop.
590  */
pci_find_next_ht_capability(struct pci_dev * dev,int pos,int ht_cap)591 int pci_find_next_ht_capability(struct pci_dev *dev, int pos, int ht_cap)
592 {
593 	return __pci_find_next_ht_cap(dev, pos + PCI_CAP_LIST_NEXT, ht_cap);
594 }
595 EXPORT_SYMBOL_GPL(pci_find_next_ht_capability);
596 
597 /**
598  * pci_find_ht_capability - query a device's Hypertransport capabilities
599  * @dev: PCI device to query
600  * @ht_cap: Hypertransport capability code
601  *
602  * Tell if a device supports a given Hypertransport capability.
603  * Returns an address within the device's PCI configuration space
604  * or 0 in case the device does not support the request capability.
605  * The address points to the PCI capability, of type PCI_CAP_ID_HT,
606  * which has a Hypertransport capability matching @ht_cap.
607  */
pci_find_ht_capability(struct pci_dev * dev,int ht_cap)608 int pci_find_ht_capability(struct pci_dev *dev, int ht_cap)
609 {
610 	int pos;
611 
612 	pos = __pci_bus_find_cap_start(dev->bus, dev->devfn, dev->hdr_type);
613 	if (pos)
614 		pos = __pci_find_next_ht_cap(dev, pos, ht_cap);
615 
616 	return pos;
617 }
618 EXPORT_SYMBOL_GPL(pci_find_ht_capability);
619 
620 /**
621  * pci_find_parent_resource - return resource region of parent bus of given region
622  * @dev: PCI device structure contains resources to be searched
623  * @res: child resource record for which parent is sought
624  *
625  *  For given resource region of given device, return the resource
626  *  region of parent bus the given region is contained in.
627  */
pci_find_parent_resource(const struct pci_dev * dev,struct resource * res)628 struct resource *pci_find_parent_resource(const struct pci_dev *dev,
629 					  struct resource *res)
630 {
631 	const struct pci_bus *bus = dev->bus;
632 	struct resource *r;
633 	int i;
634 
635 	pci_bus_for_each_resource(bus, r, i) {
636 		if (!r)
637 			continue;
638 		if (resource_contains(r, res)) {
639 
640 			/*
641 			 * If the window is prefetchable but the BAR is
642 			 * not, the allocator made a mistake.
643 			 */
644 			if (r->flags & IORESOURCE_PREFETCH &&
645 			    !(res->flags & IORESOURCE_PREFETCH))
646 				return NULL;
647 
648 			/*
649 			 * If we're below a transparent bridge, there may
650 			 * be both a positively-decoded aperture and a
651 			 * subtractively-decoded region that contain the BAR.
652 			 * We want the positively-decoded one, so this depends
653 			 * on pci_bus_for_each_resource() giving us those
654 			 * first.
655 			 */
656 			return r;
657 		}
658 	}
659 	return NULL;
660 }
661 EXPORT_SYMBOL(pci_find_parent_resource);
662 
663 /**
664  * pci_find_resource - Return matching PCI device resource
665  * @dev: PCI device to query
666  * @res: Resource to look for
667  *
668  * Goes over standard PCI resources (BARs) and checks if the given resource
669  * is partially or fully contained in any of them. In that case the
670  * matching resource is returned, %NULL otherwise.
671  */
pci_find_resource(struct pci_dev * dev,struct resource * res)672 struct resource *pci_find_resource(struct pci_dev *dev, struct resource *res)
673 {
674 	int i;
675 
676 	for (i = 0; i < PCI_ROM_RESOURCE; i++) {
677 		struct resource *r = &dev->resource[i];
678 
679 		if (r->start && resource_contains(r, res))
680 			return r;
681 	}
682 
683 	return NULL;
684 }
685 EXPORT_SYMBOL(pci_find_resource);
686 
687 /**
688  * pci_find_pcie_root_port - return PCIe Root Port
689  * @dev: PCI device to query
690  *
691  * Traverse up the parent chain and return the PCIe Root Port PCI Device
692  * for a given PCI Device.
693  */
pci_find_pcie_root_port(struct pci_dev * dev)694 struct pci_dev *pci_find_pcie_root_port(struct pci_dev *dev)
695 {
696 	struct pci_dev *bridge, *highest_pcie_bridge = dev;
697 
698 	bridge = pci_upstream_bridge(dev);
699 	while (bridge && pci_is_pcie(bridge)) {
700 		highest_pcie_bridge = bridge;
701 		bridge = pci_upstream_bridge(bridge);
702 	}
703 
704 	if (pci_pcie_type(highest_pcie_bridge) != PCI_EXP_TYPE_ROOT_PORT)
705 		return NULL;
706 
707 	return highest_pcie_bridge;
708 }
709 EXPORT_SYMBOL(pci_find_pcie_root_port);
710 
711 /**
712  * pci_wait_for_pending - wait for @mask bit(s) to clear in status word @pos
713  * @dev: the PCI device to operate on
714  * @pos: config space offset of status word
715  * @mask: mask of bit(s) to care about in status word
716  *
717  * Return 1 when mask bit(s) in status word clear, 0 otherwise.
718  */
pci_wait_for_pending(struct pci_dev * dev,int pos,u16 mask)719 int pci_wait_for_pending(struct pci_dev *dev, int pos, u16 mask)
720 {
721 	int i;
722 
723 	/* Wait for Transaction Pending bit clean */
724 	for (i = 0; i < 4; i++) {
725 		u16 status;
726 		if (i)
727 			msleep((1 << (i - 1)) * 100);
728 
729 		pci_read_config_word(dev, pos, &status);
730 		if (!(status & mask))
731 			return 1;
732 	}
733 
734 	return 0;
735 }
736 
737 /**
738  * pci_restore_bars - restore a device's BAR values (e.g. after wake-up)
739  * @dev: PCI device to have its BARs restored
740  *
741  * Restore the BAR values for a given device, so as to make it
742  * accessible by its driver.
743  */
pci_restore_bars(struct pci_dev * dev)744 static void pci_restore_bars(struct pci_dev *dev)
745 {
746 	int i;
747 
748 	for (i = 0; i < PCI_BRIDGE_RESOURCES; i++)
749 		pci_update_resource(dev, i);
750 }
751 
752 static const struct pci_platform_pm_ops *pci_platform_pm;
753 
pci_set_platform_pm(const struct pci_platform_pm_ops * ops)754 int pci_set_platform_pm(const struct pci_platform_pm_ops *ops)
755 {
756 	if (!ops->is_manageable || !ops->set_state  || !ops->get_state ||
757 	    !ops->choose_state  || !ops->set_wakeup || !ops->need_resume)
758 		return -EINVAL;
759 	pci_platform_pm = ops;
760 	return 0;
761 }
762 
platform_pci_power_manageable(struct pci_dev * dev)763 static inline bool platform_pci_power_manageable(struct pci_dev *dev)
764 {
765 	return pci_platform_pm ? pci_platform_pm->is_manageable(dev) : false;
766 }
767 
platform_pci_set_power_state(struct pci_dev * dev,pci_power_t t)768 static inline int platform_pci_set_power_state(struct pci_dev *dev,
769 					       pci_power_t t)
770 {
771 	return pci_platform_pm ? pci_platform_pm->set_state(dev, t) : -ENOSYS;
772 }
773 
platform_pci_get_power_state(struct pci_dev * dev)774 static inline pci_power_t platform_pci_get_power_state(struct pci_dev *dev)
775 {
776 	return pci_platform_pm ? pci_platform_pm->get_state(dev) : PCI_UNKNOWN;
777 }
778 
platform_pci_choose_state(struct pci_dev * dev)779 static inline pci_power_t platform_pci_choose_state(struct pci_dev *dev)
780 {
781 	return pci_platform_pm ?
782 			pci_platform_pm->choose_state(dev) : PCI_POWER_ERROR;
783 }
784 
platform_pci_set_wakeup(struct pci_dev * dev,bool enable)785 static inline int platform_pci_set_wakeup(struct pci_dev *dev, bool enable)
786 {
787 	return pci_platform_pm ?
788 			pci_platform_pm->set_wakeup(dev, enable) : -ENODEV;
789 }
790 
platform_pci_need_resume(struct pci_dev * dev)791 static inline bool platform_pci_need_resume(struct pci_dev *dev)
792 {
793 	return pci_platform_pm ? pci_platform_pm->need_resume(dev) : false;
794 }
795 
796 /**
797  * pci_raw_set_power_state - Use PCI PM registers to set the power state of
798  *                           given PCI device
799  * @dev: PCI device to handle.
800  * @state: PCI power state (D0, D1, D2, D3hot) to put the device into.
801  *
802  * RETURN VALUE:
803  * -EINVAL if the requested state is invalid.
804  * -EIO if device does not support PCI PM or its PM capabilities register has a
805  * wrong version, or device doesn't support the requested state.
806  * 0 if device already is in the requested state.
807  * 0 if device's power state has been successfully changed.
808  */
pci_raw_set_power_state(struct pci_dev * dev,pci_power_t state)809 static int pci_raw_set_power_state(struct pci_dev *dev, pci_power_t state)
810 {
811 	u16 pmcsr;
812 	bool need_restore = false;
813 
814 	/* Check if we're already there */
815 	if (dev->current_state == state)
816 		return 0;
817 
818 	if (!dev->pm_cap)
819 		return -EIO;
820 
821 	if (state < PCI_D0 || state > PCI_D3hot)
822 		return -EINVAL;
823 
824 	/* Validate current state:
825 	 * Can enter D0 from any state, but if we can only go deeper
826 	 * to sleep if we're already in a low power state
827 	 */
828 	if (state != PCI_D0 && dev->current_state <= PCI_D3cold
829 	    && dev->current_state > state) {
830 		pci_err(dev, "invalid power transition (from state %d to %d)\n",
831 			dev->current_state, state);
832 		return -EINVAL;
833 	}
834 
835 	/* check if this device supports the desired state */
836 	if ((state == PCI_D1 && !dev->d1_support)
837 	   || (state == PCI_D2 && !dev->d2_support))
838 		return -EIO;
839 
840 	pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
841 
842 	/* If we're (effectively) in D3, force entire word to 0.
843 	 * This doesn't affect PME_Status, disables PME_En, and
844 	 * sets PowerState to 0.
845 	 */
846 	switch (dev->current_state) {
847 	case PCI_D0:
848 	case PCI_D1:
849 	case PCI_D2:
850 		pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
851 		pmcsr |= state;
852 		break;
853 	case PCI_D3hot:
854 	case PCI_D3cold:
855 	case PCI_UNKNOWN: /* Boot-up */
856 		if ((pmcsr & PCI_PM_CTRL_STATE_MASK) == PCI_D3hot
857 		 && !(pmcsr & PCI_PM_CTRL_NO_SOFT_RESET))
858 			need_restore = true;
859 		/* Fall-through: force to D0 */
860 	default:
861 		pmcsr = 0;
862 		break;
863 	}
864 
865 	/* enter specified state */
866 	pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, pmcsr);
867 
868 	/* Mandatory power management transition delays */
869 	/* see PCI PM 1.1 5.6.1 table 18 */
870 	if (state == PCI_D3hot || dev->current_state == PCI_D3hot)
871 		pci_dev_d3_sleep(dev);
872 	else if (state == PCI_D2 || dev->current_state == PCI_D2)
873 		udelay(PCI_PM_D2_DELAY);
874 
875 	pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
876 	dev->current_state = (pmcsr & PCI_PM_CTRL_STATE_MASK);
877 	if (dev->current_state != state && printk_ratelimit())
878 		pci_info(dev, "Refused to change power state, currently in D%d\n",
879 			 dev->current_state);
880 
881 	/*
882 	 * According to section 5.4.1 of the "PCI BUS POWER MANAGEMENT
883 	 * INTERFACE SPECIFICATION, REV. 1.2", a device transitioning
884 	 * from D3hot to D0 _may_ perform an internal reset, thereby
885 	 * going to "D0 Uninitialized" rather than "D0 Initialized".
886 	 * For example, at least some versions of the 3c905B and the
887 	 * 3c556B exhibit this behaviour.
888 	 *
889 	 * At least some laptop BIOSen (e.g. the Thinkpad T21) leave
890 	 * devices in a D3hot state at boot.  Consequently, we need to
891 	 * restore at least the BARs so that the device will be
892 	 * accessible to its driver.
893 	 */
894 	if (need_restore)
895 		pci_restore_bars(dev);
896 
897 	if (dev->bus->self)
898 		pcie_aspm_pm_state_change(dev->bus->self);
899 
900 	return 0;
901 }
902 
903 /**
904  * pci_update_current_state - Read power state of given device and cache it
905  * @dev: PCI device to handle.
906  * @state: State to cache in case the device doesn't have the PM capability
907  *
908  * The power state is read from the PMCSR register, which however is
909  * inaccessible in D3cold.  The platform firmware is therefore queried first
910  * to detect accessibility of the register.  In case the platform firmware
911  * reports an incorrect state or the device isn't power manageable by the
912  * platform at all, we try to detect D3cold by testing accessibility of the
913  * vendor ID in config space.
914  */
pci_update_current_state(struct pci_dev * dev,pci_power_t state)915 void pci_update_current_state(struct pci_dev *dev, pci_power_t state)
916 {
917 	if (platform_pci_get_power_state(dev) == PCI_D3cold ||
918 	    !pci_device_is_present(dev)) {
919 		dev->current_state = PCI_D3cold;
920 	} else if (dev->pm_cap) {
921 		u16 pmcsr;
922 
923 		pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
924 		dev->current_state = (pmcsr & PCI_PM_CTRL_STATE_MASK);
925 	} else {
926 		dev->current_state = state;
927 	}
928 }
929 
930 /**
931  * pci_platform_power_transition - Use platform to change device power state
932  * @dev: PCI device to handle.
933  * @state: State to put the device into.
934  */
pci_platform_power_transition(struct pci_dev * dev,pci_power_t state)935 static int pci_platform_power_transition(struct pci_dev *dev, pci_power_t state)
936 {
937 	int error;
938 
939 	if (platform_pci_power_manageable(dev)) {
940 		error = platform_pci_set_power_state(dev, state);
941 		if (!error)
942 			pci_update_current_state(dev, state);
943 	} else
944 		error = -ENODEV;
945 
946 	if (error && !dev->pm_cap) /* Fall back to PCI_D0 */
947 		dev->current_state = PCI_D0;
948 
949 	return error;
950 }
951 
952 /**
953  * pci_wakeup - Wake up a PCI device
954  * @pci_dev: Device to handle.
955  * @ign: ignored parameter
956  */
pci_wakeup(struct pci_dev * pci_dev,void * ign)957 static int pci_wakeup(struct pci_dev *pci_dev, void *ign)
958 {
959 	pci_wakeup_event(pci_dev);
960 	pm_request_resume(&pci_dev->dev);
961 	return 0;
962 }
963 
964 /**
965  * pci_wakeup_bus - Walk given bus and wake up devices on it
966  * @bus: Top bus of the subtree to walk.
967  */
pci_wakeup_bus(struct pci_bus * bus)968 void pci_wakeup_bus(struct pci_bus *bus)
969 {
970 	if (bus)
971 		pci_walk_bus(bus, pci_wakeup, NULL);
972 }
973 
974 /**
975  * __pci_start_power_transition - Start power transition of a PCI device
976  * @dev: PCI device to handle.
977  * @state: State to put the device into.
978  */
__pci_start_power_transition(struct pci_dev * dev,pci_power_t state)979 static void __pci_start_power_transition(struct pci_dev *dev, pci_power_t state)
980 {
981 	if (state == PCI_D0) {
982 		pci_platform_power_transition(dev, PCI_D0);
983 		/*
984 		 * Mandatory power management transition delays, see
985 		 * PCI Express Base Specification Revision 2.0 Section
986 		 * 6.6.1: Conventional Reset.  Do not delay for
987 		 * devices powered on/off by corresponding bridge,
988 		 * because have already delayed for the bridge.
989 		 */
990 		if (dev->runtime_d3cold) {
991 			if (dev->d3cold_delay)
992 				msleep(dev->d3cold_delay);
993 			/*
994 			 * When powering on a bridge from D3cold, the
995 			 * whole hierarchy may be powered on into
996 			 * D0uninitialized state, resume them to give
997 			 * them a chance to suspend again
998 			 */
999 			pci_wakeup_bus(dev->subordinate);
1000 		}
1001 	}
1002 }
1003 
1004 /**
1005  * __pci_dev_set_current_state - Set current state of a PCI device
1006  * @dev: Device to handle
1007  * @data: pointer to state to be set
1008  */
__pci_dev_set_current_state(struct pci_dev * dev,void * data)1009 static int __pci_dev_set_current_state(struct pci_dev *dev, void *data)
1010 {
1011 	pci_power_t state = *(pci_power_t *)data;
1012 
1013 	dev->current_state = state;
1014 	return 0;
1015 }
1016 
1017 /**
1018  * pci_bus_set_current_state - Walk given bus and set current state of devices
1019  * @bus: Top bus of the subtree to walk.
1020  * @state: state to be set
1021  */
pci_bus_set_current_state(struct pci_bus * bus,pci_power_t state)1022 void pci_bus_set_current_state(struct pci_bus *bus, pci_power_t state)
1023 {
1024 	if (bus)
1025 		pci_walk_bus(bus, __pci_dev_set_current_state, &state);
1026 }
1027 
1028 /**
1029  * __pci_complete_power_transition - Complete power transition of a PCI device
1030  * @dev: PCI device to handle.
1031  * @state: State to put the device into.
1032  *
1033  * This function should not be called directly by device drivers.
1034  */
__pci_complete_power_transition(struct pci_dev * dev,pci_power_t state)1035 int __pci_complete_power_transition(struct pci_dev *dev, pci_power_t state)
1036 {
1037 	int ret;
1038 
1039 	if (state <= PCI_D0)
1040 		return -EINVAL;
1041 	ret = pci_platform_power_transition(dev, state);
1042 	/* Power off the bridge may power off the whole hierarchy */
1043 	if (!ret && state == PCI_D3cold)
1044 		pci_bus_set_current_state(dev->subordinate, PCI_D3cold);
1045 	return ret;
1046 }
1047 EXPORT_SYMBOL_GPL(__pci_complete_power_transition);
1048 
1049 /**
1050  * pci_set_power_state - Set the power state of a PCI device
1051  * @dev: PCI device to handle.
1052  * @state: PCI power state (D0, D1, D2, D3hot) to put the device into.
1053  *
1054  * Transition a device to a new power state, using the platform firmware and/or
1055  * the device's PCI PM registers.
1056  *
1057  * RETURN VALUE:
1058  * -EINVAL if the requested state is invalid.
1059  * -EIO if device does not support PCI PM or its PM capabilities register has a
1060  * wrong version, or device doesn't support the requested state.
1061  * 0 if the transition is to D1 or D2 but D1 and D2 are not supported.
1062  * 0 if device already is in the requested state.
1063  * 0 if the transition is to D3 but D3 is not supported.
1064  * 0 if device's power state has been successfully changed.
1065  */
pci_set_power_state(struct pci_dev * dev,pci_power_t state)1066 int pci_set_power_state(struct pci_dev *dev, pci_power_t state)
1067 {
1068 	int error;
1069 
1070 	/* bound the state we're entering */
1071 	if (state > PCI_D3cold)
1072 		state = PCI_D3cold;
1073 	else if (state < PCI_D0)
1074 		state = PCI_D0;
1075 	else if ((state == PCI_D1 || state == PCI_D2) && pci_no_d1d2(dev))
1076 		/*
1077 		 * If the device or the parent bridge do not support PCI PM,
1078 		 * ignore the request if we're doing anything other than putting
1079 		 * it into D0 (which would only happen on boot).
1080 		 */
1081 		return 0;
1082 
1083 	/* Check if we're already there */
1084 	if (dev->current_state == state)
1085 		return 0;
1086 
1087 	__pci_start_power_transition(dev, state);
1088 
1089 	/* This device is quirked not to be put into D3, so
1090 	   don't put it in D3 */
1091 	if (state >= PCI_D3hot && (dev->dev_flags & PCI_DEV_FLAGS_NO_D3))
1092 		return 0;
1093 
1094 	/*
1095 	 * To put device in D3cold, we put device into D3hot in native
1096 	 * way, then put device into D3cold with platform ops
1097 	 */
1098 	error = pci_raw_set_power_state(dev, state > PCI_D3hot ?
1099 					PCI_D3hot : state);
1100 
1101 	if (!__pci_complete_power_transition(dev, state))
1102 		error = 0;
1103 
1104 	return error;
1105 }
1106 EXPORT_SYMBOL(pci_set_power_state);
1107 
1108 /**
1109  * pci_power_up - Put the given device into D0 forcibly
1110  * @dev: PCI device to power up
1111  */
pci_power_up(struct pci_dev * dev)1112 void pci_power_up(struct pci_dev *dev)
1113 {
1114 	__pci_start_power_transition(dev, PCI_D0);
1115 	pci_raw_set_power_state(dev, PCI_D0);
1116 	pci_update_current_state(dev, PCI_D0);
1117 }
1118 
1119 /**
1120  * pci_choose_state - Choose the power state of a PCI device
1121  * @dev: PCI device to be suspended
1122  * @state: target sleep state for the whole system. This is the value
1123  *	that is passed to suspend() function.
1124  *
1125  * Returns PCI power state suitable for given device and given system
1126  * message.
1127  */
1128 
pci_choose_state(struct pci_dev * dev,pm_message_t state)1129 pci_power_t pci_choose_state(struct pci_dev *dev, pm_message_t state)
1130 {
1131 	pci_power_t ret;
1132 
1133 	if (!dev->pm_cap)
1134 		return PCI_D0;
1135 
1136 	ret = platform_pci_choose_state(dev);
1137 	if (ret != PCI_POWER_ERROR)
1138 		return ret;
1139 
1140 	switch (state.event) {
1141 	case PM_EVENT_ON:
1142 		return PCI_D0;
1143 	case PM_EVENT_FREEZE:
1144 	case PM_EVENT_PRETHAW:
1145 		/* REVISIT both freeze and pre-thaw "should" use D0 */
1146 	case PM_EVENT_SUSPEND:
1147 	case PM_EVENT_HIBERNATE:
1148 		return PCI_D3hot;
1149 	default:
1150 		pci_info(dev, "unrecognized suspend event %d\n",
1151 			 state.event);
1152 		BUG();
1153 	}
1154 	return PCI_D0;
1155 }
1156 EXPORT_SYMBOL(pci_choose_state);
1157 
1158 #define PCI_EXP_SAVE_REGS	7
1159 
_pci_find_saved_cap(struct pci_dev * pci_dev,u16 cap,bool extended)1160 static struct pci_cap_saved_state *_pci_find_saved_cap(struct pci_dev *pci_dev,
1161 						       u16 cap, bool extended)
1162 {
1163 	struct pci_cap_saved_state *tmp;
1164 
1165 	hlist_for_each_entry(tmp, &pci_dev->saved_cap_space, next) {
1166 		if (tmp->cap.cap_extended == extended && tmp->cap.cap_nr == cap)
1167 			return tmp;
1168 	}
1169 	return NULL;
1170 }
1171 
pci_find_saved_cap(struct pci_dev * dev,char cap)1172 struct pci_cap_saved_state *pci_find_saved_cap(struct pci_dev *dev, char cap)
1173 {
1174 	return _pci_find_saved_cap(dev, cap, false);
1175 }
1176 
pci_find_saved_ext_cap(struct pci_dev * dev,u16 cap)1177 struct pci_cap_saved_state *pci_find_saved_ext_cap(struct pci_dev *dev, u16 cap)
1178 {
1179 	return _pci_find_saved_cap(dev, cap, true);
1180 }
1181 
pci_save_pcie_state(struct pci_dev * dev)1182 static int pci_save_pcie_state(struct pci_dev *dev)
1183 {
1184 	int i = 0;
1185 	struct pci_cap_saved_state *save_state;
1186 	u16 *cap;
1187 
1188 	if (!pci_is_pcie(dev))
1189 		return 0;
1190 
1191 	save_state = pci_find_saved_cap(dev, PCI_CAP_ID_EXP);
1192 	if (!save_state) {
1193 		pci_err(dev, "buffer not found in %s\n", __func__);
1194 		return -ENOMEM;
1195 	}
1196 
1197 	cap = (u16 *)&save_state->cap.data[0];
1198 	pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &cap[i++]);
1199 	pcie_capability_read_word(dev, PCI_EXP_LNKCTL, &cap[i++]);
1200 	pcie_capability_read_word(dev, PCI_EXP_SLTCTL, &cap[i++]);
1201 	pcie_capability_read_word(dev, PCI_EXP_RTCTL,  &cap[i++]);
1202 	pcie_capability_read_word(dev, PCI_EXP_DEVCTL2, &cap[i++]);
1203 	pcie_capability_read_word(dev, PCI_EXP_LNKCTL2, &cap[i++]);
1204 	pcie_capability_read_word(dev, PCI_EXP_SLTCTL2, &cap[i++]);
1205 
1206 	return 0;
1207 }
1208 
pci_restore_pcie_state(struct pci_dev * dev)1209 static void pci_restore_pcie_state(struct pci_dev *dev)
1210 {
1211 	int i = 0;
1212 	struct pci_cap_saved_state *save_state;
1213 	u16 *cap;
1214 
1215 	save_state = pci_find_saved_cap(dev, PCI_CAP_ID_EXP);
1216 	if (!save_state)
1217 		return;
1218 
1219 	cap = (u16 *)&save_state->cap.data[0];
1220 	pcie_capability_write_word(dev, PCI_EXP_DEVCTL, cap[i++]);
1221 	pcie_capability_write_word(dev, PCI_EXP_LNKCTL, cap[i++]);
1222 	pcie_capability_write_word(dev, PCI_EXP_SLTCTL, cap[i++]);
1223 	pcie_capability_write_word(dev, PCI_EXP_RTCTL, cap[i++]);
1224 	pcie_capability_write_word(dev, PCI_EXP_DEVCTL2, cap[i++]);
1225 	pcie_capability_write_word(dev, PCI_EXP_LNKCTL2, cap[i++]);
1226 	pcie_capability_write_word(dev, PCI_EXP_SLTCTL2, cap[i++]);
1227 }
1228 
1229 
pci_save_pcix_state(struct pci_dev * dev)1230 static int pci_save_pcix_state(struct pci_dev *dev)
1231 {
1232 	int pos;
1233 	struct pci_cap_saved_state *save_state;
1234 
1235 	pos = pci_find_capability(dev, PCI_CAP_ID_PCIX);
1236 	if (!pos)
1237 		return 0;
1238 
1239 	save_state = pci_find_saved_cap(dev, PCI_CAP_ID_PCIX);
1240 	if (!save_state) {
1241 		pci_err(dev, "buffer not found in %s\n", __func__);
1242 		return -ENOMEM;
1243 	}
1244 
1245 	pci_read_config_word(dev, pos + PCI_X_CMD,
1246 			     (u16 *)save_state->cap.data);
1247 
1248 	return 0;
1249 }
1250 
pci_restore_pcix_state(struct pci_dev * dev)1251 static void pci_restore_pcix_state(struct pci_dev *dev)
1252 {
1253 	int i = 0, pos;
1254 	struct pci_cap_saved_state *save_state;
1255 	u16 *cap;
1256 
1257 	save_state = pci_find_saved_cap(dev, PCI_CAP_ID_PCIX);
1258 	pos = pci_find_capability(dev, PCI_CAP_ID_PCIX);
1259 	if (!save_state || !pos)
1260 		return;
1261 	cap = (u16 *)&save_state->cap.data[0];
1262 
1263 	pci_write_config_word(dev, pos + PCI_X_CMD, cap[i++]);
1264 }
1265 
1266 
1267 /**
1268  * pci_save_state - save the PCI configuration space of a device before suspending
1269  * @dev: - PCI device that we're dealing with
1270  */
pci_save_state(struct pci_dev * dev)1271 int pci_save_state(struct pci_dev *dev)
1272 {
1273 	int i;
1274 	/* XXX: 100% dword access ok here? */
1275 	for (i = 0; i < 16; i++)
1276 		pci_read_config_dword(dev, i * 4, &dev->saved_config_space[i]);
1277 	dev->state_saved = true;
1278 
1279 	i = pci_save_pcie_state(dev);
1280 	if (i != 0)
1281 		return i;
1282 
1283 	i = pci_save_pcix_state(dev);
1284 	if (i != 0)
1285 		return i;
1286 
1287 	return pci_save_vc_state(dev);
1288 }
1289 EXPORT_SYMBOL(pci_save_state);
1290 
pci_restore_config_dword(struct pci_dev * pdev,int offset,u32 saved_val,int retry,bool force)1291 static void pci_restore_config_dword(struct pci_dev *pdev, int offset,
1292 				     u32 saved_val, int retry, bool force)
1293 {
1294 	u32 val;
1295 
1296 	pci_read_config_dword(pdev, offset, &val);
1297 	if (!force && val == saved_val)
1298 		return;
1299 
1300 	for (;;) {
1301 		pci_dbg(pdev, "restoring config space at offset %#x (was %#x, writing %#x)\n",
1302 			offset, val, saved_val);
1303 		pci_write_config_dword(pdev, offset, saved_val);
1304 		if (retry-- <= 0)
1305 			return;
1306 
1307 		pci_read_config_dword(pdev, offset, &val);
1308 		if (val == saved_val)
1309 			return;
1310 
1311 		mdelay(1);
1312 	}
1313 }
1314 
pci_restore_config_space_range(struct pci_dev * pdev,int start,int end,int retry,bool force)1315 static void pci_restore_config_space_range(struct pci_dev *pdev,
1316 					   int start, int end, int retry,
1317 					   bool force)
1318 {
1319 	int index;
1320 
1321 	for (index = end; index >= start; index--)
1322 		pci_restore_config_dword(pdev, 4 * index,
1323 					 pdev->saved_config_space[index],
1324 					 retry, force);
1325 }
1326 
pci_restore_config_space(struct pci_dev * pdev)1327 static void pci_restore_config_space(struct pci_dev *pdev)
1328 {
1329 	if (pdev->hdr_type == PCI_HEADER_TYPE_NORMAL) {
1330 		pci_restore_config_space_range(pdev, 10, 15, 0, false);
1331 		/* Restore BARs before the command register. */
1332 		pci_restore_config_space_range(pdev, 4, 9, 10, false);
1333 		pci_restore_config_space_range(pdev, 0, 3, 0, false);
1334 	} else if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE) {
1335 		pci_restore_config_space_range(pdev, 12, 15, 0, false);
1336 
1337 		/*
1338 		 * Force rewriting of prefetch registers to avoid S3 resume
1339 		 * issues on Intel PCI bridges that occur when these
1340 		 * registers are not explicitly written.
1341 		 */
1342 		pci_restore_config_space_range(pdev, 9, 11, 0, true);
1343 		pci_restore_config_space_range(pdev, 0, 8, 0, false);
1344 	} else {
1345 		pci_restore_config_space_range(pdev, 0, 15, 0, false);
1346 	}
1347 }
1348 
pci_restore_rebar_state(struct pci_dev * pdev)1349 static void pci_restore_rebar_state(struct pci_dev *pdev)
1350 {
1351 	unsigned int pos, nbars, i;
1352 	u32 ctrl;
1353 
1354 	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_REBAR);
1355 	if (!pos)
1356 		return;
1357 
1358 	pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
1359 	nbars = (ctrl & PCI_REBAR_CTRL_NBAR_MASK) >>
1360 		    PCI_REBAR_CTRL_NBAR_SHIFT;
1361 
1362 	for (i = 0; i < nbars; i++, pos += 8) {
1363 		struct resource *res;
1364 		int bar_idx, size;
1365 
1366 		pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
1367 		bar_idx = ctrl & PCI_REBAR_CTRL_BAR_IDX;
1368 		res = pdev->resource + bar_idx;
1369 		size = ilog2(resource_size(res)) - 20;
1370 		ctrl &= ~PCI_REBAR_CTRL_BAR_SIZE;
1371 		ctrl |= size << PCI_REBAR_CTRL_BAR_SHIFT;
1372 		pci_write_config_dword(pdev, pos + PCI_REBAR_CTRL, ctrl);
1373 	}
1374 }
1375 
1376 /**
1377  * pci_restore_state - Restore the saved state of a PCI device
1378  * @dev: - PCI device that we're dealing with
1379  */
pci_restore_state(struct pci_dev * dev)1380 void pci_restore_state(struct pci_dev *dev)
1381 {
1382 	if (!dev->state_saved)
1383 		return;
1384 
1385 	/* PCI Express register must be restored first */
1386 	pci_restore_pcie_state(dev);
1387 	pci_restore_pasid_state(dev);
1388 	pci_restore_pri_state(dev);
1389 	pci_restore_ats_state(dev);
1390 	pci_restore_vc_state(dev);
1391 	pci_restore_rebar_state(dev);
1392 
1393 	pci_cleanup_aer_error_status_regs(dev);
1394 
1395 	pci_restore_config_space(dev);
1396 
1397 	pci_restore_pcix_state(dev);
1398 	pci_restore_msi_state(dev);
1399 
1400 	/* Restore ACS and IOV configuration state */
1401 	pci_enable_acs(dev);
1402 	pci_restore_iov_state(dev);
1403 
1404 	dev->state_saved = false;
1405 }
1406 EXPORT_SYMBOL(pci_restore_state);
1407 
1408 struct pci_saved_state {
1409 	u32 config_space[16];
1410 	struct pci_cap_saved_data cap[0];
1411 };
1412 
1413 /**
1414  * pci_store_saved_state - Allocate and return an opaque struct containing
1415  *			   the device saved state.
1416  * @dev: PCI device that we're dealing with
1417  *
1418  * Return NULL if no state or error.
1419  */
pci_store_saved_state(struct pci_dev * dev)1420 struct pci_saved_state *pci_store_saved_state(struct pci_dev *dev)
1421 {
1422 	struct pci_saved_state *state;
1423 	struct pci_cap_saved_state *tmp;
1424 	struct pci_cap_saved_data *cap;
1425 	size_t size;
1426 
1427 	if (!dev->state_saved)
1428 		return NULL;
1429 
1430 	size = sizeof(*state) + sizeof(struct pci_cap_saved_data);
1431 
1432 	hlist_for_each_entry(tmp, &dev->saved_cap_space, next)
1433 		size += sizeof(struct pci_cap_saved_data) + tmp->cap.size;
1434 
1435 	state = kzalloc(size, GFP_KERNEL);
1436 	if (!state)
1437 		return NULL;
1438 
1439 	memcpy(state->config_space, dev->saved_config_space,
1440 	       sizeof(state->config_space));
1441 
1442 	cap = state->cap;
1443 	hlist_for_each_entry(tmp, &dev->saved_cap_space, next) {
1444 		size_t len = sizeof(struct pci_cap_saved_data) + tmp->cap.size;
1445 		memcpy(cap, &tmp->cap, len);
1446 		cap = (struct pci_cap_saved_data *)((u8 *)cap + len);
1447 	}
1448 	/* Empty cap_save terminates list */
1449 
1450 	return state;
1451 }
1452 EXPORT_SYMBOL_GPL(pci_store_saved_state);
1453 
1454 /**
1455  * pci_load_saved_state - Reload the provided save state into struct pci_dev.
1456  * @dev: PCI device that we're dealing with
1457  * @state: Saved state returned from pci_store_saved_state()
1458  */
pci_load_saved_state(struct pci_dev * dev,struct pci_saved_state * state)1459 int pci_load_saved_state(struct pci_dev *dev,
1460 			 struct pci_saved_state *state)
1461 {
1462 	struct pci_cap_saved_data *cap;
1463 
1464 	dev->state_saved = false;
1465 
1466 	if (!state)
1467 		return 0;
1468 
1469 	memcpy(dev->saved_config_space, state->config_space,
1470 	       sizeof(state->config_space));
1471 
1472 	cap = state->cap;
1473 	while (cap->size) {
1474 		struct pci_cap_saved_state *tmp;
1475 
1476 		tmp = _pci_find_saved_cap(dev, cap->cap_nr, cap->cap_extended);
1477 		if (!tmp || tmp->cap.size != cap->size)
1478 			return -EINVAL;
1479 
1480 		memcpy(tmp->cap.data, cap->data, tmp->cap.size);
1481 		cap = (struct pci_cap_saved_data *)((u8 *)cap +
1482 		       sizeof(struct pci_cap_saved_data) + cap->size);
1483 	}
1484 
1485 	dev->state_saved = true;
1486 	return 0;
1487 }
1488 EXPORT_SYMBOL_GPL(pci_load_saved_state);
1489 
1490 /**
1491  * pci_load_and_free_saved_state - Reload the save state pointed to by state,
1492  *				   and free the memory allocated for it.
1493  * @dev: PCI device that we're dealing with
1494  * @state: Pointer to saved state returned from pci_store_saved_state()
1495  */
pci_load_and_free_saved_state(struct pci_dev * dev,struct pci_saved_state ** state)1496 int pci_load_and_free_saved_state(struct pci_dev *dev,
1497 				  struct pci_saved_state **state)
1498 {
1499 	int ret = pci_load_saved_state(dev, *state);
1500 	kfree(*state);
1501 	*state = NULL;
1502 	return ret;
1503 }
1504 EXPORT_SYMBOL_GPL(pci_load_and_free_saved_state);
1505 
pcibios_enable_device(struct pci_dev * dev,int bars)1506 int __weak pcibios_enable_device(struct pci_dev *dev, int bars)
1507 {
1508 	return pci_enable_resources(dev, bars);
1509 }
1510 
do_pci_enable_device(struct pci_dev * dev,int bars)1511 static int do_pci_enable_device(struct pci_dev *dev, int bars)
1512 {
1513 	int err;
1514 	struct pci_dev *bridge;
1515 	u16 cmd;
1516 	u8 pin;
1517 
1518 	err = pci_set_power_state(dev, PCI_D0);
1519 	if (err < 0 && err != -EIO)
1520 		return err;
1521 
1522 	bridge = pci_upstream_bridge(dev);
1523 	if (bridge)
1524 		pcie_aspm_powersave_config_link(bridge);
1525 
1526 	err = pcibios_enable_device(dev, bars);
1527 	if (err < 0)
1528 		return err;
1529 	pci_fixup_device(pci_fixup_enable, dev);
1530 
1531 	if (dev->msi_enabled || dev->msix_enabled)
1532 		return 0;
1533 
1534 	pci_read_config_byte(dev, PCI_INTERRUPT_PIN, &pin);
1535 	if (pin) {
1536 		pci_read_config_word(dev, PCI_COMMAND, &cmd);
1537 		if (cmd & PCI_COMMAND_INTX_DISABLE)
1538 			pci_write_config_word(dev, PCI_COMMAND,
1539 					      cmd & ~PCI_COMMAND_INTX_DISABLE);
1540 	}
1541 
1542 	return 0;
1543 }
1544 
1545 /**
1546  * pci_reenable_device - Resume abandoned device
1547  * @dev: PCI device to be resumed
1548  *
1549  *  Note this function is a backend of pci_default_resume and is not supposed
1550  *  to be called by normal code, write proper resume handler and use it instead.
1551  */
pci_reenable_device(struct pci_dev * dev)1552 int pci_reenable_device(struct pci_dev *dev)
1553 {
1554 	if (pci_is_enabled(dev))
1555 		return do_pci_enable_device(dev, (1 << PCI_NUM_RESOURCES) - 1);
1556 	return 0;
1557 }
1558 EXPORT_SYMBOL(pci_reenable_device);
1559 
pci_enable_bridge(struct pci_dev * dev)1560 static void pci_enable_bridge(struct pci_dev *dev)
1561 {
1562 	struct pci_dev *bridge;
1563 	int retval;
1564 
1565 	bridge = pci_upstream_bridge(dev);
1566 	if (bridge)
1567 		pci_enable_bridge(bridge);
1568 
1569 	if (pci_is_enabled(dev)) {
1570 		if (!dev->is_busmaster)
1571 			pci_set_master(dev);
1572 		return;
1573 	}
1574 
1575 	retval = pci_enable_device(dev);
1576 	if (retval)
1577 		pci_err(dev, "Error enabling bridge (%d), continuing\n",
1578 			retval);
1579 	pci_set_master(dev);
1580 }
1581 
pci_enable_device_flags(struct pci_dev * dev,unsigned long flags)1582 static int pci_enable_device_flags(struct pci_dev *dev, unsigned long flags)
1583 {
1584 	struct pci_dev *bridge;
1585 	int err;
1586 	int i, bars = 0;
1587 
1588 	/*
1589 	 * Power state could be unknown at this point, either due to a fresh
1590 	 * boot or a device removal call.  So get the current power state
1591 	 * so that things like MSI message writing will behave as expected
1592 	 * (e.g. if the device really is in D0 at enable time).
1593 	 */
1594 	pci_update_current_state(dev, dev->current_state);
1595 
1596 	if (atomic_inc_return(&dev->enable_cnt) > 1)
1597 		return 0;		/* already enabled */
1598 
1599 	bridge = pci_upstream_bridge(dev);
1600 	if (bridge)
1601 		pci_enable_bridge(bridge);
1602 
1603 	/* only skip sriov related */
1604 	for (i = 0; i <= PCI_ROM_RESOURCE; i++)
1605 		if (dev->resource[i].flags & flags)
1606 			bars |= (1 << i);
1607 	for (i = PCI_BRIDGE_RESOURCES; i < DEVICE_COUNT_RESOURCE; i++)
1608 		if (dev->resource[i].flags & flags)
1609 			bars |= (1 << i);
1610 
1611 	err = do_pci_enable_device(dev, bars);
1612 	if (err < 0)
1613 		atomic_dec(&dev->enable_cnt);
1614 	return err;
1615 }
1616 
1617 /**
1618  * pci_enable_device_io - Initialize a device for use with IO space
1619  * @dev: PCI device to be initialized
1620  *
1621  *  Initialize device before it's used by a driver. Ask low-level code
1622  *  to enable I/O resources. Wake up the device if it was suspended.
1623  *  Beware, this function can fail.
1624  */
pci_enable_device_io(struct pci_dev * dev)1625 int pci_enable_device_io(struct pci_dev *dev)
1626 {
1627 	return pci_enable_device_flags(dev, IORESOURCE_IO);
1628 }
1629 EXPORT_SYMBOL(pci_enable_device_io);
1630 
1631 /**
1632  * pci_enable_device_mem - Initialize a device for use with Memory space
1633  * @dev: PCI device to be initialized
1634  *
1635  *  Initialize device before it's used by a driver. Ask low-level code
1636  *  to enable Memory resources. Wake up the device if it was suspended.
1637  *  Beware, this function can fail.
1638  */
pci_enable_device_mem(struct pci_dev * dev)1639 int pci_enable_device_mem(struct pci_dev *dev)
1640 {
1641 	return pci_enable_device_flags(dev, IORESOURCE_MEM);
1642 }
1643 EXPORT_SYMBOL(pci_enable_device_mem);
1644 
1645 /**
1646  * pci_enable_device - Initialize device before it's used by a driver.
1647  * @dev: PCI device to be initialized
1648  *
1649  *  Initialize device before it's used by a driver. Ask low-level code
1650  *  to enable I/O and memory. Wake up the device if it was suspended.
1651  *  Beware, this function can fail.
1652  *
1653  *  Note we don't actually enable the device many times if we call
1654  *  this function repeatedly (we just increment the count).
1655  */
pci_enable_device(struct pci_dev * dev)1656 int pci_enable_device(struct pci_dev *dev)
1657 {
1658 	return pci_enable_device_flags(dev, IORESOURCE_MEM | IORESOURCE_IO);
1659 }
1660 EXPORT_SYMBOL(pci_enable_device);
1661 
1662 /*
1663  * Managed PCI resources.  This manages device on/off, intx/msi/msix
1664  * on/off and BAR regions.  pci_dev itself records msi/msix status, so
1665  * there's no need to track it separately.  pci_devres is initialized
1666  * when a device is enabled using managed PCI device enable interface.
1667  */
1668 struct pci_devres {
1669 	unsigned int enabled:1;
1670 	unsigned int pinned:1;
1671 	unsigned int orig_intx:1;
1672 	unsigned int restore_intx:1;
1673 	unsigned int mwi:1;
1674 	u32 region_mask;
1675 };
1676 
pcim_release(struct device * gendev,void * res)1677 static void pcim_release(struct device *gendev, void *res)
1678 {
1679 	struct pci_dev *dev = to_pci_dev(gendev);
1680 	struct pci_devres *this = res;
1681 	int i;
1682 
1683 	if (dev->msi_enabled)
1684 		pci_disable_msi(dev);
1685 	if (dev->msix_enabled)
1686 		pci_disable_msix(dev);
1687 
1688 	for (i = 0; i < DEVICE_COUNT_RESOURCE; i++)
1689 		if (this->region_mask & (1 << i))
1690 			pci_release_region(dev, i);
1691 
1692 	if (this->mwi)
1693 		pci_clear_mwi(dev);
1694 
1695 	if (this->restore_intx)
1696 		pci_intx(dev, this->orig_intx);
1697 
1698 	if (this->enabled && !this->pinned)
1699 		pci_disable_device(dev);
1700 }
1701 
get_pci_dr(struct pci_dev * pdev)1702 static struct pci_devres *get_pci_dr(struct pci_dev *pdev)
1703 {
1704 	struct pci_devres *dr, *new_dr;
1705 
1706 	dr = devres_find(&pdev->dev, pcim_release, NULL, NULL);
1707 	if (dr)
1708 		return dr;
1709 
1710 	new_dr = devres_alloc(pcim_release, sizeof(*new_dr), GFP_KERNEL);
1711 	if (!new_dr)
1712 		return NULL;
1713 	return devres_get(&pdev->dev, new_dr, NULL, NULL);
1714 }
1715 
find_pci_dr(struct pci_dev * pdev)1716 static struct pci_devres *find_pci_dr(struct pci_dev *pdev)
1717 {
1718 	if (pci_is_managed(pdev))
1719 		return devres_find(&pdev->dev, pcim_release, NULL, NULL);
1720 	return NULL;
1721 }
1722 
1723 /**
1724  * pcim_enable_device - Managed pci_enable_device()
1725  * @pdev: PCI device to be initialized
1726  *
1727  * Managed pci_enable_device().
1728  */
pcim_enable_device(struct pci_dev * pdev)1729 int pcim_enable_device(struct pci_dev *pdev)
1730 {
1731 	struct pci_devres *dr;
1732 	int rc;
1733 
1734 	dr = get_pci_dr(pdev);
1735 	if (unlikely(!dr))
1736 		return -ENOMEM;
1737 	if (dr->enabled)
1738 		return 0;
1739 
1740 	rc = pci_enable_device(pdev);
1741 	if (!rc) {
1742 		pdev->is_managed = 1;
1743 		dr->enabled = 1;
1744 	}
1745 	return rc;
1746 }
1747 EXPORT_SYMBOL(pcim_enable_device);
1748 
1749 /**
1750  * pcim_pin_device - Pin managed PCI device
1751  * @pdev: PCI device to pin
1752  *
1753  * Pin managed PCI device @pdev.  Pinned device won't be disabled on
1754  * driver detach.  @pdev must have been enabled with
1755  * pcim_enable_device().
1756  */
pcim_pin_device(struct pci_dev * pdev)1757 void pcim_pin_device(struct pci_dev *pdev)
1758 {
1759 	struct pci_devres *dr;
1760 
1761 	dr = find_pci_dr(pdev);
1762 	WARN_ON(!dr || !dr->enabled);
1763 	if (dr)
1764 		dr->pinned = 1;
1765 }
1766 EXPORT_SYMBOL(pcim_pin_device);
1767 
1768 /*
1769  * pcibios_add_device - provide arch specific hooks when adding device dev
1770  * @dev: the PCI device being added
1771  *
1772  * Permits the platform to provide architecture specific functionality when
1773  * devices are added. This is the default implementation. Architecture
1774  * implementations can override this.
1775  */
pcibios_add_device(struct pci_dev * dev)1776 int __weak pcibios_add_device(struct pci_dev *dev)
1777 {
1778 	return 0;
1779 }
1780 
1781 /**
1782  * pcibios_release_device - provide arch specific hooks when releasing device dev
1783  * @dev: the PCI device being released
1784  *
1785  * Permits the platform to provide architecture specific functionality when
1786  * devices are released. This is the default implementation. Architecture
1787  * implementations can override this.
1788  */
pcibios_release_device(struct pci_dev * dev)1789 void __weak pcibios_release_device(struct pci_dev *dev) {}
1790 
1791 /**
1792  * pcibios_disable_device - disable arch specific PCI resources for device dev
1793  * @dev: the PCI device to disable
1794  *
1795  * Disables architecture specific PCI resources for the device. This
1796  * is the default implementation. Architecture implementations can
1797  * override this.
1798  */
pcibios_disable_device(struct pci_dev * dev)1799 void __weak pcibios_disable_device(struct pci_dev *dev) {}
1800 
1801 /**
1802  * pcibios_penalize_isa_irq - penalize an ISA IRQ
1803  * @irq: ISA IRQ to penalize
1804  * @active: IRQ active or not
1805  *
1806  * Permits the platform to provide architecture-specific functionality when
1807  * penalizing ISA IRQs. This is the default implementation. Architecture
1808  * implementations can override this.
1809  */
pcibios_penalize_isa_irq(int irq,int active)1810 void __weak pcibios_penalize_isa_irq(int irq, int active) {}
1811 
do_pci_disable_device(struct pci_dev * dev)1812 static void do_pci_disable_device(struct pci_dev *dev)
1813 {
1814 	u16 pci_command;
1815 
1816 	pci_read_config_word(dev, PCI_COMMAND, &pci_command);
1817 	if (pci_command & PCI_COMMAND_MASTER) {
1818 		pci_command &= ~PCI_COMMAND_MASTER;
1819 		pci_write_config_word(dev, PCI_COMMAND, pci_command);
1820 	}
1821 
1822 	pcibios_disable_device(dev);
1823 }
1824 
1825 /**
1826  * pci_disable_enabled_device - Disable device without updating enable_cnt
1827  * @dev: PCI device to disable
1828  *
1829  * NOTE: This function is a backend of PCI power management routines and is
1830  * not supposed to be called drivers.
1831  */
pci_disable_enabled_device(struct pci_dev * dev)1832 void pci_disable_enabled_device(struct pci_dev *dev)
1833 {
1834 	if (pci_is_enabled(dev))
1835 		do_pci_disable_device(dev);
1836 }
1837 
1838 /**
1839  * pci_disable_device - Disable PCI device after use
1840  * @dev: PCI device to be disabled
1841  *
1842  * Signal to the system that the PCI device is not in use by the system
1843  * anymore.  This only involves disabling PCI bus-mastering, if active.
1844  *
1845  * Note we don't actually disable the device until all callers of
1846  * pci_enable_device() have called pci_disable_device().
1847  */
pci_disable_device(struct pci_dev * dev)1848 void pci_disable_device(struct pci_dev *dev)
1849 {
1850 	struct pci_devres *dr;
1851 
1852 	dr = find_pci_dr(dev);
1853 	if (dr)
1854 		dr->enabled = 0;
1855 
1856 	dev_WARN_ONCE(&dev->dev, atomic_read(&dev->enable_cnt) <= 0,
1857 		      "disabling already-disabled device");
1858 
1859 	if (atomic_dec_return(&dev->enable_cnt) != 0)
1860 		return;
1861 
1862 	do_pci_disable_device(dev);
1863 
1864 	dev->is_busmaster = 0;
1865 }
1866 EXPORT_SYMBOL(pci_disable_device);
1867 
1868 /**
1869  * pcibios_set_pcie_reset_state - set reset state for device dev
1870  * @dev: the PCIe device reset
1871  * @state: Reset state to enter into
1872  *
1873  *
1874  * Sets the PCIe reset state for the device. This is the default
1875  * implementation. Architecture implementations can override this.
1876  */
pcibios_set_pcie_reset_state(struct pci_dev * dev,enum pcie_reset_state state)1877 int __weak pcibios_set_pcie_reset_state(struct pci_dev *dev,
1878 					enum pcie_reset_state state)
1879 {
1880 	return -EINVAL;
1881 }
1882 
1883 /**
1884  * pci_set_pcie_reset_state - set reset state for device dev
1885  * @dev: the PCIe device reset
1886  * @state: Reset state to enter into
1887  *
1888  *
1889  * Sets the PCI reset state for the device.
1890  */
pci_set_pcie_reset_state(struct pci_dev * dev,enum pcie_reset_state state)1891 int pci_set_pcie_reset_state(struct pci_dev *dev, enum pcie_reset_state state)
1892 {
1893 	return pcibios_set_pcie_reset_state(dev, state);
1894 }
1895 EXPORT_SYMBOL_GPL(pci_set_pcie_reset_state);
1896 
1897 /**
1898  * pcie_clear_root_pme_status - Clear root port PME interrupt status.
1899  * @dev: PCIe root port or event collector.
1900  */
pcie_clear_root_pme_status(struct pci_dev * dev)1901 void pcie_clear_root_pme_status(struct pci_dev *dev)
1902 {
1903 	pcie_capability_set_dword(dev, PCI_EXP_RTSTA, PCI_EXP_RTSTA_PME);
1904 }
1905 
1906 /**
1907  * pci_check_pme_status - Check if given device has generated PME.
1908  * @dev: Device to check.
1909  *
1910  * Check the PME status of the device and if set, clear it and clear PME enable
1911  * (if set).  Return 'true' if PME status and PME enable were both set or
1912  * 'false' otherwise.
1913  */
pci_check_pme_status(struct pci_dev * dev)1914 bool pci_check_pme_status(struct pci_dev *dev)
1915 {
1916 	int pmcsr_pos;
1917 	u16 pmcsr;
1918 	bool ret = false;
1919 
1920 	if (!dev->pm_cap)
1921 		return false;
1922 
1923 	pmcsr_pos = dev->pm_cap + PCI_PM_CTRL;
1924 	pci_read_config_word(dev, pmcsr_pos, &pmcsr);
1925 	if (!(pmcsr & PCI_PM_CTRL_PME_STATUS))
1926 		return false;
1927 
1928 	/* Clear PME status. */
1929 	pmcsr |= PCI_PM_CTRL_PME_STATUS;
1930 	if (pmcsr & PCI_PM_CTRL_PME_ENABLE) {
1931 		/* Disable PME to avoid interrupt flood. */
1932 		pmcsr &= ~PCI_PM_CTRL_PME_ENABLE;
1933 		ret = true;
1934 	}
1935 
1936 	pci_write_config_word(dev, pmcsr_pos, pmcsr);
1937 
1938 	return ret;
1939 }
1940 
1941 /**
1942  * pci_pme_wakeup - Wake up a PCI device if its PME Status bit is set.
1943  * @dev: Device to handle.
1944  * @pme_poll_reset: Whether or not to reset the device's pme_poll flag.
1945  *
1946  * Check if @dev has generated PME and queue a resume request for it in that
1947  * case.
1948  */
pci_pme_wakeup(struct pci_dev * dev,void * pme_poll_reset)1949 static int pci_pme_wakeup(struct pci_dev *dev, void *pme_poll_reset)
1950 {
1951 	if (pme_poll_reset && dev->pme_poll)
1952 		dev->pme_poll = false;
1953 
1954 	if (pci_check_pme_status(dev)) {
1955 		pci_wakeup_event(dev);
1956 		pm_request_resume(&dev->dev);
1957 	}
1958 	return 0;
1959 }
1960 
1961 /**
1962  * pci_pme_wakeup_bus - Walk given bus and wake up devices on it, if necessary.
1963  * @bus: Top bus of the subtree to walk.
1964  */
pci_pme_wakeup_bus(struct pci_bus * bus)1965 void pci_pme_wakeup_bus(struct pci_bus *bus)
1966 {
1967 	if (bus)
1968 		pci_walk_bus(bus, pci_pme_wakeup, (void *)true);
1969 }
1970 
1971 
1972 /**
1973  * pci_pme_capable - check the capability of PCI device to generate PME#
1974  * @dev: PCI device to handle.
1975  * @state: PCI state from which device will issue PME#.
1976  */
pci_pme_capable(struct pci_dev * dev,pci_power_t state)1977 bool pci_pme_capable(struct pci_dev *dev, pci_power_t state)
1978 {
1979 	if (!dev->pm_cap)
1980 		return false;
1981 
1982 	return !!(dev->pme_support & (1 << state));
1983 }
1984 EXPORT_SYMBOL(pci_pme_capable);
1985 
pci_pme_list_scan(struct work_struct * work)1986 static void pci_pme_list_scan(struct work_struct *work)
1987 {
1988 	struct pci_pme_device *pme_dev, *n;
1989 
1990 	mutex_lock(&pci_pme_list_mutex);
1991 	list_for_each_entry_safe(pme_dev, n, &pci_pme_list, list) {
1992 		if (pme_dev->dev->pme_poll) {
1993 			struct pci_dev *bridge;
1994 
1995 			bridge = pme_dev->dev->bus->self;
1996 			/*
1997 			 * If bridge is in low power state, the
1998 			 * configuration space of subordinate devices
1999 			 * may be not accessible
2000 			 */
2001 			if (bridge && bridge->current_state != PCI_D0)
2002 				continue;
2003 			/*
2004 			 * If the device is in D3cold it should not be
2005 			 * polled either.
2006 			 */
2007 			if (pme_dev->dev->current_state == PCI_D3cold)
2008 				continue;
2009 
2010 			pci_pme_wakeup(pme_dev->dev, NULL);
2011 		} else {
2012 			list_del(&pme_dev->list);
2013 			kfree(pme_dev);
2014 		}
2015 	}
2016 	if (!list_empty(&pci_pme_list))
2017 		queue_delayed_work(system_freezable_wq, &pci_pme_work,
2018 				   msecs_to_jiffies(PME_TIMEOUT));
2019 	mutex_unlock(&pci_pme_list_mutex);
2020 }
2021 
__pci_pme_active(struct pci_dev * dev,bool enable)2022 static void __pci_pme_active(struct pci_dev *dev, bool enable)
2023 {
2024 	u16 pmcsr;
2025 
2026 	if (!dev->pme_support)
2027 		return;
2028 
2029 	pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
2030 	/* Clear PME_Status by writing 1 to it and enable PME# */
2031 	pmcsr |= PCI_PM_CTRL_PME_STATUS | PCI_PM_CTRL_PME_ENABLE;
2032 	if (!enable)
2033 		pmcsr &= ~PCI_PM_CTRL_PME_ENABLE;
2034 
2035 	pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, pmcsr);
2036 }
2037 
2038 /**
2039  * pci_pme_restore - Restore PME configuration after config space restore.
2040  * @dev: PCI device to update.
2041  */
pci_pme_restore(struct pci_dev * dev)2042 void pci_pme_restore(struct pci_dev *dev)
2043 {
2044 	u16 pmcsr;
2045 
2046 	if (!dev->pme_support)
2047 		return;
2048 
2049 	pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
2050 	if (dev->wakeup_prepared) {
2051 		pmcsr |= PCI_PM_CTRL_PME_ENABLE;
2052 		pmcsr &= ~PCI_PM_CTRL_PME_STATUS;
2053 	} else {
2054 		pmcsr &= ~PCI_PM_CTRL_PME_ENABLE;
2055 		pmcsr |= PCI_PM_CTRL_PME_STATUS;
2056 	}
2057 	pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, pmcsr);
2058 }
2059 
2060 /**
2061  * pci_pme_active - enable or disable PCI device's PME# function
2062  * @dev: PCI device to handle.
2063  * @enable: 'true' to enable PME# generation; 'false' to disable it.
2064  *
2065  * The caller must verify that the device is capable of generating PME# before
2066  * calling this function with @enable equal to 'true'.
2067  */
pci_pme_active(struct pci_dev * dev,bool enable)2068 void pci_pme_active(struct pci_dev *dev, bool enable)
2069 {
2070 	__pci_pme_active(dev, enable);
2071 
2072 	/*
2073 	 * PCI (as opposed to PCIe) PME requires that the device have
2074 	 * its PME# line hooked up correctly. Not all hardware vendors
2075 	 * do this, so the PME never gets delivered and the device
2076 	 * remains asleep. The easiest way around this is to
2077 	 * periodically walk the list of suspended devices and check
2078 	 * whether any have their PME flag set. The assumption is that
2079 	 * we'll wake up often enough anyway that this won't be a huge
2080 	 * hit, and the power savings from the devices will still be a
2081 	 * win.
2082 	 *
2083 	 * Although PCIe uses in-band PME message instead of PME# line
2084 	 * to report PME, PME does not work for some PCIe devices in
2085 	 * reality.  For example, there are devices that set their PME
2086 	 * status bits, but don't really bother to send a PME message;
2087 	 * there are PCI Express Root Ports that don't bother to
2088 	 * trigger interrupts when they receive PME messages from the
2089 	 * devices below.  So PME poll is used for PCIe devices too.
2090 	 */
2091 
2092 	if (dev->pme_poll) {
2093 		struct pci_pme_device *pme_dev;
2094 		if (enable) {
2095 			pme_dev = kmalloc(sizeof(struct pci_pme_device),
2096 					  GFP_KERNEL);
2097 			if (!pme_dev) {
2098 				pci_warn(dev, "can't enable PME#\n");
2099 				return;
2100 			}
2101 			pme_dev->dev = dev;
2102 			mutex_lock(&pci_pme_list_mutex);
2103 			list_add(&pme_dev->list, &pci_pme_list);
2104 			if (list_is_singular(&pci_pme_list))
2105 				queue_delayed_work(system_freezable_wq,
2106 						   &pci_pme_work,
2107 						   msecs_to_jiffies(PME_TIMEOUT));
2108 			mutex_unlock(&pci_pme_list_mutex);
2109 		} else {
2110 			mutex_lock(&pci_pme_list_mutex);
2111 			list_for_each_entry(pme_dev, &pci_pme_list, list) {
2112 				if (pme_dev->dev == dev) {
2113 					list_del(&pme_dev->list);
2114 					kfree(pme_dev);
2115 					break;
2116 				}
2117 			}
2118 			mutex_unlock(&pci_pme_list_mutex);
2119 		}
2120 	}
2121 
2122 	pci_dbg(dev, "PME# %s\n", enable ? "enabled" : "disabled");
2123 }
2124 EXPORT_SYMBOL(pci_pme_active);
2125 
2126 /**
2127  * __pci_enable_wake - enable PCI device as wakeup event source
2128  * @dev: PCI device affected
2129  * @state: PCI state from which device will issue wakeup events
2130  * @enable: True to enable event generation; false to disable
2131  *
2132  * This enables the device as a wakeup event source, or disables it.
2133  * When such events involves platform-specific hooks, those hooks are
2134  * called automatically by this routine.
2135  *
2136  * Devices with legacy power management (no standard PCI PM capabilities)
2137  * always require such platform hooks.
2138  *
2139  * RETURN VALUE:
2140  * 0 is returned on success
2141  * -EINVAL is returned if device is not supposed to wake up the system
2142  * Error code depending on the platform is returned if both the platform and
2143  * the native mechanism fail to enable the generation of wake-up events
2144  */
__pci_enable_wake(struct pci_dev * dev,pci_power_t state,bool enable)2145 static int __pci_enable_wake(struct pci_dev *dev, pci_power_t state, bool enable)
2146 {
2147 	int ret = 0;
2148 
2149 	/*
2150 	 * Bridges can only signal wakeup on behalf of subordinate devices,
2151 	 * but that is set up elsewhere, so skip them.
2152 	 */
2153 	if (pci_has_subordinate(dev))
2154 		return 0;
2155 
2156 	/* Don't do the same thing twice in a row for one device. */
2157 	if (!!enable == !!dev->wakeup_prepared)
2158 		return 0;
2159 
2160 	/*
2161 	 * According to "PCI System Architecture" 4th ed. by Tom Shanley & Don
2162 	 * Anderson we should be doing PME# wake enable followed by ACPI wake
2163 	 * enable.  To disable wake-up we call the platform first, for symmetry.
2164 	 */
2165 
2166 	if (enable) {
2167 		int error;
2168 
2169 		/*
2170 		 * Enable PME signaling if the device can signal PME from
2171 		 * D3cold regardless of whether or not it can signal PME from
2172 		 * the current target state, because that will allow it to
2173 		 * signal PME when the hierarchy above it goes into D3cold and
2174 		 * the device itself ends up in D3cold as a result of that.
2175 		 */
2176 		if (pci_pme_capable(dev, state) || pci_pme_capable(dev, PCI_D3cold))
2177 			pci_pme_active(dev, true);
2178 		else
2179 			ret = 1;
2180 		error = platform_pci_set_wakeup(dev, true);
2181 		if (ret)
2182 			ret = error;
2183 		if (!ret)
2184 			dev->wakeup_prepared = true;
2185 	} else {
2186 		platform_pci_set_wakeup(dev, false);
2187 		pci_pme_active(dev, false);
2188 		dev->wakeup_prepared = false;
2189 	}
2190 
2191 	return ret;
2192 }
2193 
2194 /**
2195  * pci_enable_wake - change wakeup settings for a PCI device
2196  * @pci_dev: Target device
2197  * @state: PCI state from which device will issue wakeup events
2198  * @enable: Whether or not to enable event generation
2199  *
2200  * If @enable is set, check device_may_wakeup() for the device before calling
2201  * __pci_enable_wake() for it.
2202  */
pci_enable_wake(struct pci_dev * pci_dev,pci_power_t state,bool enable)2203 int pci_enable_wake(struct pci_dev *pci_dev, pci_power_t state, bool enable)
2204 {
2205 	if (enable && !device_may_wakeup(&pci_dev->dev))
2206 		return -EINVAL;
2207 
2208 	return __pci_enable_wake(pci_dev, state, enable);
2209 }
2210 EXPORT_SYMBOL(pci_enable_wake);
2211 
2212 /**
2213  * pci_wake_from_d3 - enable/disable device to wake up from D3_hot or D3_cold
2214  * @dev: PCI device to prepare
2215  * @enable: True to enable wake-up event generation; false to disable
2216  *
2217  * Many drivers want the device to wake up the system from D3_hot or D3_cold
2218  * and this function allows them to set that up cleanly - pci_enable_wake()
2219  * should not be called twice in a row to enable wake-up due to PCI PM vs ACPI
2220  * ordering constraints.
2221  *
2222  * This function only returns error code if the device is not allowed to wake
2223  * up the system from sleep or it is not capable of generating PME# from both
2224  * D3_hot and D3_cold and the platform is unable to enable wake-up power for it.
2225  */
pci_wake_from_d3(struct pci_dev * dev,bool enable)2226 int pci_wake_from_d3(struct pci_dev *dev, bool enable)
2227 {
2228 	return pci_pme_capable(dev, PCI_D3cold) ?
2229 			pci_enable_wake(dev, PCI_D3cold, enable) :
2230 			pci_enable_wake(dev, PCI_D3hot, enable);
2231 }
2232 EXPORT_SYMBOL(pci_wake_from_d3);
2233 
2234 /**
2235  * pci_target_state - find an appropriate low power state for a given PCI dev
2236  * @dev: PCI device
2237  * @wakeup: Whether or not wakeup functionality will be enabled for the device.
2238  *
2239  * Use underlying platform code to find a supported low power state for @dev.
2240  * If the platform can't manage @dev, return the deepest state from which it
2241  * can generate wake events, based on any available PME info.
2242  */
pci_target_state(struct pci_dev * dev,bool wakeup)2243 static pci_power_t pci_target_state(struct pci_dev *dev, bool wakeup)
2244 {
2245 	pci_power_t target_state = PCI_D3hot;
2246 
2247 	if (platform_pci_power_manageable(dev)) {
2248 		/*
2249 		 * Call the platform to find the target state for the device.
2250 		 */
2251 		pci_power_t state = platform_pci_choose_state(dev);
2252 
2253 		switch (state) {
2254 		case PCI_POWER_ERROR:
2255 		case PCI_UNKNOWN:
2256 			break;
2257 		case PCI_D1:
2258 		case PCI_D2:
2259 			if (pci_no_d1d2(dev))
2260 				break;
2261 			/* else: fall through */
2262 		default:
2263 			target_state = state;
2264 		}
2265 
2266 		return target_state;
2267 	}
2268 
2269 	if (!dev->pm_cap)
2270 		target_state = PCI_D0;
2271 
2272 	/*
2273 	 * If the device is in D3cold even though it's not power-manageable by
2274 	 * the platform, it may have been powered down by non-standard means.
2275 	 * Best to let it slumber.
2276 	 */
2277 	if (dev->current_state == PCI_D3cold)
2278 		target_state = PCI_D3cold;
2279 
2280 	if (wakeup && dev->pme_support) {
2281 		pci_power_t state = target_state;
2282 
2283 		/*
2284 		 * Find the deepest state from which the device can generate
2285 		 * PME#.
2286 		 */
2287 		while (state && !(dev->pme_support & (1 << state)))
2288 			state--;
2289 
2290 		if (state)
2291 			return state;
2292 		else if (dev->pme_support & 1)
2293 			return PCI_D0;
2294 	}
2295 
2296 	return target_state;
2297 }
2298 
2299 /**
2300  * pci_prepare_to_sleep - prepare PCI device for system-wide transition into a sleep state
2301  * @dev: Device to handle.
2302  *
2303  * Choose the power state appropriate for the device depending on whether
2304  * it can wake up the system and/or is power manageable by the platform
2305  * (PCI_D3hot is the default) and put the device into that state.
2306  */
pci_prepare_to_sleep(struct pci_dev * dev)2307 int pci_prepare_to_sleep(struct pci_dev *dev)
2308 {
2309 	bool wakeup = device_may_wakeup(&dev->dev);
2310 	pci_power_t target_state = pci_target_state(dev, wakeup);
2311 	int error;
2312 
2313 	if (target_state == PCI_POWER_ERROR)
2314 		return -EIO;
2315 
2316 	pci_enable_wake(dev, target_state, wakeup);
2317 
2318 	error = pci_set_power_state(dev, target_state);
2319 
2320 	if (error)
2321 		pci_enable_wake(dev, target_state, false);
2322 
2323 	return error;
2324 }
2325 EXPORT_SYMBOL(pci_prepare_to_sleep);
2326 
2327 /**
2328  * pci_back_from_sleep - turn PCI device on during system-wide transition into working state
2329  * @dev: Device to handle.
2330  *
2331  * Disable device's system wake-up capability and put it into D0.
2332  */
pci_back_from_sleep(struct pci_dev * dev)2333 int pci_back_from_sleep(struct pci_dev *dev)
2334 {
2335 	pci_enable_wake(dev, PCI_D0, false);
2336 	return pci_set_power_state(dev, PCI_D0);
2337 }
2338 EXPORT_SYMBOL(pci_back_from_sleep);
2339 
2340 /**
2341  * pci_finish_runtime_suspend - Carry out PCI-specific part of runtime suspend.
2342  * @dev: PCI device being suspended.
2343  *
2344  * Prepare @dev to generate wake-up events at run time and put it into a low
2345  * power state.
2346  */
pci_finish_runtime_suspend(struct pci_dev * dev)2347 int pci_finish_runtime_suspend(struct pci_dev *dev)
2348 {
2349 	pci_power_t target_state;
2350 	int error;
2351 
2352 	target_state = pci_target_state(dev, device_can_wakeup(&dev->dev));
2353 	if (target_state == PCI_POWER_ERROR)
2354 		return -EIO;
2355 
2356 	dev->runtime_d3cold = target_state == PCI_D3cold;
2357 
2358 	__pci_enable_wake(dev, target_state, pci_dev_run_wake(dev));
2359 
2360 	error = pci_set_power_state(dev, target_state);
2361 
2362 	if (error) {
2363 		pci_enable_wake(dev, target_state, false);
2364 		dev->runtime_d3cold = false;
2365 	}
2366 
2367 	return error;
2368 }
2369 
2370 /**
2371  * pci_dev_run_wake - Check if device can generate run-time wake-up events.
2372  * @dev: Device to check.
2373  *
2374  * Return true if the device itself is capable of generating wake-up events
2375  * (through the platform or using the native PCIe PME) or if the device supports
2376  * PME and one of its upstream bridges can generate wake-up events.
2377  */
pci_dev_run_wake(struct pci_dev * dev)2378 bool pci_dev_run_wake(struct pci_dev *dev)
2379 {
2380 	struct pci_bus *bus = dev->bus;
2381 
2382 	if (!dev->pme_support)
2383 		return false;
2384 
2385 	/* PME-capable in principle, but not from the target power state */
2386 	if (!pci_pme_capable(dev, pci_target_state(dev, true)))
2387 		return false;
2388 
2389 	if (device_can_wakeup(&dev->dev))
2390 		return true;
2391 
2392 	while (bus->parent) {
2393 		struct pci_dev *bridge = bus->self;
2394 
2395 		if (device_can_wakeup(&bridge->dev))
2396 			return true;
2397 
2398 		bus = bus->parent;
2399 	}
2400 
2401 	/* We have reached the root bus. */
2402 	if (bus->bridge)
2403 		return device_can_wakeup(bus->bridge);
2404 
2405 	return false;
2406 }
2407 EXPORT_SYMBOL_GPL(pci_dev_run_wake);
2408 
2409 /**
2410  * pci_dev_keep_suspended - Check if the device can stay in the suspended state.
2411  * @pci_dev: Device to check.
2412  *
2413  * Return 'true' if the device is runtime-suspended, it doesn't have to be
2414  * reconfigured due to wakeup settings difference between system and runtime
2415  * suspend and the current power state of it is suitable for the upcoming
2416  * (system) transition.
2417  *
2418  * If the device is not configured for system wakeup, disable PME for it before
2419  * returning 'true' to prevent it from waking up the system unnecessarily.
2420  */
pci_dev_keep_suspended(struct pci_dev * pci_dev)2421 bool pci_dev_keep_suspended(struct pci_dev *pci_dev)
2422 {
2423 	struct device *dev = &pci_dev->dev;
2424 	bool wakeup = device_may_wakeup(dev);
2425 
2426 	if (!pm_runtime_suspended(dev)
2427 	    || pci_target_state(pci_dev, wakeup) != pci_dev->current_state
2428 	    || platform_pci_need_resume(pci_dev))
2429 		return false;
2430 
2431 	/*
2432 	 * At this point the device is good to go unless it's been configured
2433 	 * to generate PME at the runtime suspend time, but it is not supposed
2434 	 * to wake up the system.  In that case, simply disable PME for it
2435 	 * (it will have to be re-enabled on exit from system resume).
2436 	 *
2437 	 * If the device's power state is D3cold and the platform check above
2438 	 * hasn't triggered, the device's configuration is suitable and we don't
2439 	 * need to manipulate it at all.
2440 	 */
2441 	spin_lock_irq(&dev->power.lock);
2442 
2443 	if (pm_runtime_suspended(dev) && pci_dev->current_state < PCI_D3cold &&
2444 	    !wakeup)
2445 		__pci_pme_active(pci_dev, false);
2446 
2447 	spin_unlock_irq(&dev->power.lock);
2448 	return true;
2449 }
2450 
2451 /**
2452  * pci_dev_complete_resume - Finalize resume from system sleep for a device.
2453  * @pci_dev: Device to handle.
2454  *
2455  * If the device is runtime suspended and wakeup-capable, enable PME for it as
2456  * it might have been disabled during the prepare phase of system suspend if
2457  * the device was not configured for system wakeup.
2458  */
pci_dev_complete_resume(struct pci_dev * pci_dev)2459 void pci_dev_complete_resume(struct pci_dev *pci_dev)
2460 {
2461 	struct device *dev = &pci_dev->dev;
2462 
2463 	if (!pci_dev_run_wake(pci_dev))
2464 		return;
2465 
2466 	spin_lock_irq(&dev->power.lock);
2467 
2468 	if (pm_runtime_suspended(dev) && pci_dev->current_state < PCI_D3cold)
2469 		__pci_pme_active(pci_dev, true);
2470 
2471 	spin_unlock_irq(&dev->power.lock);
2472 }
2473 
pci_config_pm_runtime_get(struct pci_dev * pdev)2474 void pci_config_pm_runtime_get(struct pci_dev *pdev)
2475 {
2476 	struct device *dev = &pdev->dev;
2477 	struct device *parent = dev->parent;
2478 
2479 	if (parent)
2480 		pm_runtime_get_sync(parent);
2481 	pm_runtime_get_noresume(dev);
2482 	/*
2483 	 * pdev->current_state is set to PCI_D3cold during suspending,
2484 	 * so wait until suspending completes
2485 	 */
2486 	pm_runtime_barrier(dev);
2487 	/*
2488 	 * Only need to resume devices in D3cold, because config
2489 	 * registers are still accessible for devices suspended but
2490 	 * not in D3cold.
2491 	 */
2492 	if (pdev->current_state == PCI_D3cold)
2493 		pm_runtime_resume(dev);
2494 }
2495 
pci_config_pm_runtime_put(struct pci_dev * pdev)2496 void pci_config_pm_runtime_put(struct pci_dev *pdev)
2497 {
2498 	struct device *dev = &pdev->dev;
2499 	struct device *parent = dev->parent;
2500 
2501 	pm_runtime_put(dev);
2502 	if (parent)
2503 		pm_runtime_put_sync(parent);
2504 }
2505 
2506 static const struct dmi_system_id bridge_d3_blacklist[] = {
2507 #ifdef CONFIG_X86
2508 	{
2509 		/*
2510 		 * Gigabyte X299 root port is not marked as hotplug capable
2511 		 * which allows Linux to power manage it.  However, this
2512 		 * confuses the BIOS SMI handler so don't power manage root
2513 		 * ports on that system.
2514 		 */
2515 		.ident = "X299 DESIGNARE EX-CF",
2516 		.matches = {
2517 			DMI_MATCH(DMI_BOARD_VENDOR, "Gigabyte Technology Co., Ltd."),
2518 			DMI_MATCH(DMI_BOARD_NAME, "X299 DESIGNARE EX-CF"),
2519 		},
2520 	},
2521 	{
2522 		/*
2523 		 * Downstream device is not accessible after putting a root port
2524 		 * into D3cold and back into D0 on Elo Continental Z2 board
2525 		 */
2526 		.ident = "Elo Continental Z2",
2527 		.matches = {
2528 			DMI_MATCH(DMI_BOARD_VENDOR, "Elo Touch Solutions"),
2529 			DMI_MATCH(DMI_BOARD_NAME, "Geminilake"),
2530 			DMI_MATCH(DMI_BOARD_VERSION, "Continental Z2"),
2531 		},
2532 	},
2533 #endif
2534 	{ }
2535 };
2536 
2537 /**
2538  * pci_bridge_d3_possible - Is it possible to put the bridge into D3
2539  * @bridge: Bridge to check
2540  *
2541  * This function checks if it is possible to move the bridge to D3.
2542  * Currently we only allow D3 for recent enough PCIe ports and Thunderbolt.
2543  */
pci_bridge_d3_possible(struct pci_dev * bridge)2544 bool pci_bridge_d3_possible(struct pci_dev *bridge)
2545 {
2546 	if (!pci_is_pcie(bridge))
2547 		return false;
2548 
2549 	switch (pci_pcie_type(bridge)) {
2550 	case PCI_EXP_TYPE_ROOT_PORT:
2551 	case PCI_EXP_TYPE_UPSTREAM:
2552 	case PCI_EXP_TYPE_DOWNSTREAM:
2553 		if (pci_bridge_d3_disable)
2554 			return false;
2555 
2556 		/*
2557 		 * Hotplug ports handled by firmware in System Management Mode
2558 		 * may not be put into D3 by the OS (Thunderbolt on non-Macs).
2559 		 */
2560 		if (bridge->is_hotplug_bridge && !pciehp_is_native(bridge))
2561 			return false;
2562 
2563 		if (pci_bridge_d3_force)
2564 			return true;
2565 
2566 		/* Even the oldest 2010 Thunderbolt controller supports D3. */
2567 		if (bridge->is_thunderbolt)
2568 			return true;
2569 
2570 		/*
2571 		 * Hotplug ports handled natively by the OS were not validated
2572 		 * by vendors for runtime D3 at least until 2018 because there
2573 		 * was no OS support.
2574 		 */
2575 		if (bridge->is_hotplug_bridge)
2576 			return false;
2577 
2578 		if (dmi_check_system(bridge_d3_blacklist))
2579 			return false;
2580 
2581 		/*
2582 		 * It should be safe to put PCIe ports from 2015 or newer
2583 		 * to D3.
2584 		 */
2585 		if (dmi_get_bios_year() >= 2015)
2586 			return true;
2587 		break;
2588 	}
2589 
2590 	return false;
2591 }
2592 
pci_dev_check_d3cold(struct pci_dev * dev,void * data)2593 static int pci_dev_check_d3cold(struct pci_dev *dev, void *data)
2594 {
2595 	bool *d3cold_ok = data;
2596 
2597 	if (/* The device needs to be allowed to go D3cold ... */
2598 	    dev->no_d3cold || !dev->d3cold_allowed ||
2599 
2600 	    /* ... and if it is wakeup capable to do so from D3cold. */
2601 	    (device_may_wakeup(&dev->dev) &&
2602 	     !pci_pme_capable(dev, PCI_D3cold)) ||
2603 
2604 	    /* If it is a bridge it must be allowed to go to D3. */
2605 	    !pci_power_manageable(dev))
2606 
2607 		*d3cold_ok = false;
2608 
2609 	return !*d3cold_ok;
2610 }
2611 
2612 /*
2613  * pci_bridge_d3_update - Update bridge D3 capabilities
2614  * @dev: PCI device which is changed
2615  *
2616  * Update upstream bridge PM capabilities accordingly depending on if the
2617  * device PM configuration was changed or the device is being removed.  The
2618  * change is also propagated upstream.
2619  */
pci_bridge_d3_update(struct pci_dev * dev)2620 void pci_bridge_d3_update(struct pci_dev *dev)
2621 {
2622 	bool remove = !device_is_registered(&dev->dev);
2623 	struct pci_dev *bridge;
2624 	bool d3cold_ok = true;
2625 
2626 	bridge = pci_upstream_bridge(dev);
2627 	if (!bridge || !pci_bridge_d3_possible(bridge))
2628 		return;
2629 
2630 	/*
2631 	 * If D3 is currently allowed for the bridge, removing one of its
2632 	 * children won't change that.
2633 	 */
2634 	if (remove && bridge->bridge_d3)
2635 		return;
2636 
2637 	/*
2638 	 * If D3 is currently allowed for the bridge and a child is added or
2639 	 * changed, disallowance of D3 can only be caused by that child, so
2640 	 * we only need to check that single device, not any of its siblings.
2641 	 *
2642 	 * If D3 is currently not allowed for the bridge, checking the device
2643 	 * first may allow us to skip checking its siblings.
2644 	 */
2645 	if (!remove)
2646 		pci_dev_check_d3cold(dev, &d3cold_ok);
2647 
2648 	/*
2649 	 * If D3 is currently not allowed for the bridge, this may be caused
2650 	 * either by the device being changed/removed or any of its siblings,
2651 	 * so we need to go through all children to find out if one of them
2652 	 * continues to block D3.
2653 	 */
2654 	if (d3cold_ok && !bridge->bridge_d3)
2655 		pci_walk_bus(bridge->subordinate, pci_dev_check_d3cold,
2656 			     &d3cold_ok);
2657 
2658 	if (bridge->bridge_d3 != d3cold_ok) {
2659 		bridge->bridge_d3 = d3cold_ok;
2660 		/* Propagate change to upstream bridges */
2661 		pci_bridge_d3_update(bridge);
2662 	}
2663 }
2664 
2665 /**
2666  * pci_d3cold_enable - Enable D3cold for device
2667  * @dev: PCI device to handle
2668  *
2669  * This function can be used in drivers to enable D3cold from the device
2670  * they handle.  It also updates upstream PCI bridge PM capabilities
2671  * accordingly.
2672  */
pci_d3cold_enable(struct pci_dev * dev)2673 void pci_d3cold_enable(struct pci_dev *dev)
2674 {
2675 	if (dev->no_d3cold) {
2676 		dev->no_d3cold = false;
2677 		pci_bridge_d3_update(dev);
2678 	}
2679 }
2680 EXPORT_SYMBOL_GPL(pci_d3cold_enable);
2681 
2682 /**
2683  * pci_d3cold_disable - Disable D3cold for device
2684  * @dev: PCI device to handle
2685  *
2686  * This function can be used in drivers to disable D3cold from the device
2687  * they handle.  It also updates upstream PCI bridge PM capabilities
2688  * accordingly.
2689  */
pci_d3cold_disable(struct pci_dev * dev)2690 void pci_d3cold_disable(struct pci_dev *dev)
2691 {
2692 	if (!dev->no_d3cold) {
2693 		dev->no_d3cold = true;
2694 		pci_bridge_d3_update(dev);
2695 	}
2696 }
2697 EXPORT_SYMBOL_GPL(pci_d3cold_disable);
2698 
2699 /**
2700  * pci_pm_init - Initialize PM functions of given PCI device
2701  * @dev: PCI device to handle.
2702  */
pci_pm_init(struct pci_dev * dev)2703 void pci_pm_init(struct pci_dev *dev)
2704 {
2705 	int pm;
2706 	u16 pmc;
2707 
2708 	pm_runtime_forbid(&dev->dev);
2709 	pm_runtime_set_active(&dev->dev);
2710 	pm_runtime_enable(&dev->dev);
2711 	device_enable_async_suspend(&dev->dev);
2712 	dev->wakeup_prepared = false;
2713 
2714 	dev->pm_cap = 0;
2715 	dev->pme_support = 0;
2716 
2717 	/* find PCI PM capability in list */
2718 	pm = pci_find_capability(dev, PCI_CAP_ID_PM);
2719 	if (!pm)
2720 		return;
2721 	/* Check device's ability to generate PME# */
2722 	pci_read_config_word(dev, pm + PCI_PM_PMC, &pmc);
2723 
2724 	if ((pmc & PCI_PM_CAP_VER_MASK) > 3) {
2725 		pci_err(dev, "unsupported PM cap regs version (%u)\n",
2726 			pmc & PCI_PM_CAP_VER_MASK);
2727 		return;
2728 	}
2729 
2730 	dev->pm_cap = pm;
2731 	dev->d3_delay = PCI_PM_D3_WAIT;
2732 	dev->d3cold_delay = PCI_PM_D3COLD_WAIT;
2733 	dev->bridge_d3 = pci_bridge_d3_possible(dev);
2734 	dev->d3cold_allowed = true;
2735 
2736 	dev->d1_support = false;
2737 	dev->d2_support = false;
2738 	if (!pci_no_d1d2(dev)) {
2739 		if (pmc & PCI_PM_CAP_D1)
2740 			dev->d1_support = true;
2741 		if (pmc & PCI_PM_CAP_D2)
2742 			dev->d2_support = true;
2743 
2744 		if (dev->d1_support || dev->d2_support)
2745 			pci_printk(KERN_DEBUG, dev, "supports%s%s\n",
2746 				   dev->d1_support ? " D1" : "",
2747 				   dev->d2_support ? " D2" : "");
2748 	}
2749 
2750 	pmc &= PCI_PM_CAP_PME_MASK;
2751 	if (pmc) {
2752 		pci_printk(KERN_DEBUG, dev, "PME# supported from%s%s%s%s%s\n",
2753 			 (pmc & PCI_PM_CAP_PME_D0) ? " D0" : "",
2754 			 (pmc & PCI_PM_CAP_PME_D1) ? " D1" : "",
2755 			 (pmc & PCI_PM_CAP_PME_D2) ? " D2" : "",
2756 			 (pmc & PCI_PM_CAP_PME_D3) ? " D3hot" : "",
2757 			 (pmc & PCI_PM_CAP_PME_D3cold) ? " D3cold" : "");
2758 		dev->pme_support = pmc >> PCI_PM_CAP_PME_SHIFT;
2759 		dev->pme_poll = true;
2760 		/*
2761 		 * Make device's PM flags reflect the wake-up capability, but
2762 		 * let the user space enable it to wake up the system as needed.
2763 		 */
2764 		device_set_wakeup_capable(&dev->dev, true);
2765 		/* Disable the PME# generation functionality */
2766 		pci_pme_active(dev, false);
2767 	}
2768 }
2769 
pci_ea_flags(struct pci_dev * dev,u8 prop)2770 static unsigned long pci_ea_flags(struct pci_dev *dev, u8 prop)
2771 {
2772 	unsigned long flags = IORESOURCE_PCI_FIXED | IORESOURCE_PCI_EA_BEI;
2773 
2774 	switch (prop) {
2775 	case PCI_EA_P_MEM:
2776 	case PCI_EA_P_VF_MEM:
2777 		flags |= IORESOURCE_MEM;
2778 		break;
2779 	case PCI_EA_P_MEM_PREFETCH:
2780 	case PCI_EA_P_VF_MEM_PREFETCH:
2781 		flags |= IORESOURCE_MEM | IORESOURCE_PREFETCH;
2782 		break;
2783 	case PCI_EA_P_IO:
2784 		flags |= IORESOURCE_IO;
2785 		break;
2786 	default:
2787 		return 0;
2788 	}
2789 
2790 	return flags;
2791 }
2792 
pci_ea_get_resource(struct pci_dev * dev,u8 bei,u8 prop)2793 static struct resource *pci_ea_get_resource(struct pci_dev *dev, u8 bei,
2794 					    u8 prop)
2795 {
2796 	if (bei <= PCI_EA_BEI_BAR5 && prop <= PCI_EA_P_IO)
2797 		return &dev->resource[bei];
2798 #ifdef CONFIG_PCI_IOV
2799 	else if (bei >= PCI_EA_BEI_VF_BAR0 && bei <= PCI_EA_BEI_VF_BAR5 &&
2800 		 (prop == PCI_EA_P_VF_MEM || prop == PCI_EA_P_VF_MEM_PREFETCH))
2801 		return &dev->resource[PCI_IOV_RESOURCES +
2802 				      bei - PCI_EA_BEI_VF_BAR0];
2803 #endif
2804 	else if (bei == PCI_EA_BEI_ROM)
2805 		return &dev->resource[PCI_ROM_RESOURCE];
2806 	else
2807 		return NULL;
2808 }
2809 
2810 /* Read an Enhanced Allocation (EA) entry */
pci_ea_read(struct pci_dev * dev,int offset)2811 static int pci_ea_read(struct pci_dev *dev, int offset)
2812 {
2813 	struct resource *res;
2814 	int ent_size, ent_offset = offset;
2815 	resource_size_t start, end;
2816 	unsigned long flags;
2817 	u32 dw0, bei, base, max_offset;
2818 	u8 prop;
2819 	bool support_64 = (sizeof(resource_size_t) >= 8);
2820 
2821 	pci_read_config_dword(dev, ent_offset, &dw0);
2822 	ent_offset += 4;
2823 
2824 	/* Entry size field indicates DWORDs after 1st */
2825 	ent_size = ((dw0 & PCI_EA_ES) + 1) << 2;
2826 
2827 	if (!(dw0 & PCI_EA_ENABLE)) /* Entry not enabled */
2828 		goto out;
2829 
2830 	bei = (dw0 & PCI_EA_BEI) >> 4;
2831 	prop = (dw0 & PCI_EA_PP) >> 8;
2832 
2833 	/*
2834 	 * If the Property is in the reserved range, try the Secondary
2835 	 * Property instead.
2836 	 */
2837 	if (prop > PCI_EA_P_BRIDGE_IO && prop < PCI_EA_P_MEM_RESERVED)
2838 		prop = (dw0 & PCI_EA_SP) >> 16;
2839 	if (prop > PCI_EA_P_BRIDGE_IO)
2840 		goto out;
2841 
2842 	res = pci_ea_get_resource(dev, bei, prop);
2843 	if (!res) {
2844 		pci_err(dev, "Unsupported EA entry BEI: %u\n", bei);
2845 		goto out;
2846 	}
2847 
2848 	flags = pci_ea_flags(dev, prop);
2849 	if (!flags) {
2850 		pci_err(dev, "Unsupported EA properties: %#x\n", prop);
2851 		goto out;
2852 	}
2853 
2854 	/* Read Base */
2855 	pci_read_config_dword(dev, ent_offset, &base);
2856 	start = (base & PCI_EA_FIELD_MASK);
2857 	ent_offset += 4;
2858 
2859 	/* Read MaxOffset */
2860 	pci_read_config_dword(dev, ent_offset, &max_offset);
2861 	ent_offset += 4;
2862 
2863 	/* Read Base MSBs (if 64-bit entry) */
2864 	if (base & PCI_EA_IS_64) {
2865 		u32 base_upper;
2866 
2867 		pci_read_config_dword(dev, ent_offset, &base_upper);
2868 		ent_offset += 4;
2869 
2870 		flags |= IORESOURCE_MEM_64;
2871 
2872 		/* entry starts above 32-bit boundary, can't use */
2873 		if (!support_64 && base_upper)
2874 			goto out;
2875 
2876 		if (support_64)
2877 			start |= ((u64)base_upper << 32);
2878 	}
2879 
2880 	end = start + (max_offset | 0x03);
2881 
2882 	/* Read MaxOffset MSBs (if 64-bit entry) */
2883 	if (max_offset & PCI_EA_IS_64) {
2884 		u32 max_offset_upper;
2885 
2886 		pci_read_config_dword(dev, ent_offset, &max_offset_upper);
2887 		ent_offset += 4;
2888 
2889 		flags |= IORESOURCE_MEM_64;
2890 
2891 		/* entry too big, can't use */
2892 		if (!support_64 && max_offset_upper)
2893 			goto out;
2894 
2895 		if (support_64)
2896 			end += ((u64)max_offset_upper << 32);
2897 	}
2898 
2899 	if (end < start) {
2900 		pci_err(dev, "EA Entry crosses address boundary\n");
2901 		goto out;
2902 	}
2903 
2904 	if (ent_size != ent_offset - offset) {
2905 		pci_err(dev, "EA Entry Size (%d) does not match length read (%d)\n",
2906 			ent_size, ent_offset - offset);
2907 		goto out;
2908 	}
2909 
2910 	res->name = pci_name(dev);
2911 	res->start = start;
2912 	res->end = end;
2913 	res->flags = flags;
2914 
2915 	if (bei <= PCI_EA_BEI_BAR5)
2916 		pci_printk(KERN_DEBUG, dev, "BAR %d: %pR (from Enhanced Allocation, properties %#02x)\n",
2917 			   bei, res, prop);
2918 	else if (bei == PCI_EA_BEI_ROM)
2919 		pci_printk(KERN_DEBUG, dev, "ROM: %pR (from Enhanced Allocation, properties %#02x)\n",
2920 			   res, prop);
2921 	else if (bei >= PCI_EA_BEI_VF_BAR0 && bei <= PCI_EA_BEI_VF_BAR5)
2922 		pci_printk(KERN_DEBUG, dev, "VF BAR %d: %pR (from Enhanced Allocation, properties %#02x)\n",
2923 			   bei - PCI_EA_BEI_VF_BAR0, res, prop);
2924 	else
2925 		pci_printk(KERN_DEBUG, dev, "BEI %d res: %pR (from Enhanced Allocation, properties %#02x)\n",
2926 			   bei, res, prop);
2927 
2928 out:
2929 	return offset + ent_size;
2930 }
2931 
2932 /* Enhanced Allocation Initialization */
pci_ea_init(struct pci_dev * dev)2933 void pci_ea_init(struct pci_dev *dev)
2934 {
2935 	int ea;
2936 	u8 num_ent;
2937 	int offset;
2938 	int i;
2939 
2940 	/* find PCI EA capability in list */
2941 	ea = pci_find_capability(dev, PCI_CAP_ID_EA);
2942 	if (!ea)
2943 		return;
2944 
2945 	/* determine the number of entries */
2946 	pci_bus_read_config_byte(dev->bus, dev->devfn, ea + PCI_EA_NUM_ENT,
2947 					&num_ent);
2948 	num_ent &= PCI_EA_NUM_ENT_MASK;
2949 
2950 	offset = ea + PCI_EA_FIRST_ENT;
2951 
2952 	/* Skip DWORD 2 for type 1 functions */
2953 	if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE)
2954 		offset += 4;
2955 
2956 	/* parse each EA entry */
2957 	for (i = 0; i < num_ent; ++i)
2958 		offset = pci_ea_read(dev, offset);
2959 }
2960 
pci_add_saved_cap(struct pci_dev * pci_dev,struct pci_cap_saved_state * new_cap)2961 static void pci_add_saved_cap(struct pci_dev *pci_dev,
2962 	struct pci_cap_saved_state *new_cap)
2963 {
2964 	hlist_add_head(&new_cap->next, &pci_dev->saved_cap_space);
2965 }
2966 
2967 /**
2968  * _pci_add_cap_save_buffer - allocate buffer for saving given
2969  *                            capability registers
2970  * @dev: the PCI device
2971  * @cap: the capability to allocate the buffer for
2972  * @extended: Standard or Extended capability ID
2973  * @size: requested size of the buffer
2974  */
_pci_add_cap_save_buffer(struct pci_dev * dev,u16 cap,bool extended,unsigned int size)2975 static int _pci_add_cap_save_buffer(struct pci_dev *dev, u16 cap,
2976 				    bool extended, unsigned int size)
2977 {
2978 	int pos;
2979 	struct pci_cap_saved_state *save_state;
2980 
2981 	if (extended)
2982 		pos = pci_find_ext_capability(dev, cap);
2983 	else
2984 		pos = pci_find_capability(dev, cap);
2985 
2986 	if (!pos)
2987 		return 0;
2988 
2989 	save_state = kzalloc(sizeof(*save_state) + size, GFP_KERNEL);
2990 	if (!save_state)
2991 		return -ENOMEM;
2992 
2993 	save_state->cap.cap_nr = cap;
2994 	save_state->cap.cap_extended = extended;
2995 	save_state->cap.size = size;
2996 	pci_add_saved_cap(dev, save_state);
2997 
2998 	return 0;
2999 }
3000 
pci_add_cap_save_buffer(struct pci_dev * dev,char cap,unsigned int size)3001 int pci_add_cap_save_buffer(struct pci_dev *dev, char cap, unsigned int size)
3002 {
3003 	return _pci_add_cap_save_buffer(dev, cap, false, size);
3004 }
3005 
pci_add_ext_cap_save_buffer(struct pci_dev * dev,u16 cap,unsigned int size)3006 int pci_add_ext_cap_save_buffer(struct pci_dev *dev, u16 cap, unsigned int size)
3007 {
3008 	return _pci_add_cap_save_buffer(dev, cap, true, size);
3009 }
3010 
3011 /**
3012  * pci_allocate_cap_save_buffers - allocate buffers for saving capabilities
3013  * @dev: the PCI device
3014  */
pci_allocate_cap_save_buffers(struct pci_dev * dev)3015 void pci_allocate_cap_save_buffers(struct pci_dev *dev)
3016 {
3017 	int error;
3018 
3019 	error = pci_add_cap_save_buffer(dev, PCI_CAP_ID_EXP,
3020 					PCI_EXP_SAVE_REGS * sizeof(u16));
3021 	if (error)
3022 		pci_err(dev, "unable to preallocate PCI Express save buffer\n");
3023 
3024 	error = pci_add_cap_save_buffer(dev, PCI_CAP_ID_PCIX, sizeof(u16));
3025 	if (error)
3026 		pci_err(dev, "unable to preallocate PCI-X save buffer\n");
3027 
3028 	pci_allocate_vc_save_buffers(dev);
3029 }
3030 
pci_free_cap_save_buffers(struct pci_dev * dev)3031 void pci_free_cap_save_buffers(struct pci_dev *dev)
3032 {
3033 	struct pci_cap_saved_state *tmp;
3034 	struct hlist_node *n;
3035 
3036 	hlist_for_each_entry_safe(tmp, n, &dev->saved_cap_space, next)
3037 		kfree(tmp);
3038 }
3039 
3040 /**
3041  * pci_configure_ari - enable or disable ARI forwarding
3042  * @dev: the PCI device
3043  *
3044  * If @dev and its upstream bridge both support ARI, enable ARI in the
3045  * bridge.  Otherwise, disable ARI in the bridge.
3046  */
pci_configure_ari(struct pci_dev * dev)3047 void pci_configure_ari(struct pci_dev *dev)
3048 {
3049 	u32 cap;
3050 	struct pci_dev *bridge;
3051 
3052 	if (pcie_ari_disabled || !pci_is_pcie(dev) || dev->devfn)
3053 		return;
3054 
3055 	bridge = dev->bus->self;
3056 	if (!bridge)
3057 		return;
3058 
3059 	pcie_capability_read_dword(bridge, PCI_EXP_DEVCAP2, &cap);
3060 	if (!(cap & PCI_EXP_DEVCAP2_ARI))
3061 		return;
3062 
3063 	if (pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ARI)) {
3064 		pcie_capability_set_word(bridge, PCI_EXP_DEVCTL2,
3065 					 PCI_EXP_DEVCTL2_ARI);
3066 		bridge->ari_enabled = 1;
3067 	} else {
3068 		pcie_capability_clear_word(bridge, PCI_EXP_DEVCTL2,
3069 					   PCI_EXP_DEVCTL2_ARI);
3070 		bridge->ari_enabled = 0;
3071 	}
3072 }
3073 
3074 static int pci_acs_enable;
3075 
3076 /**
3077  * pci_request_acs - ask for ACS to be enabled if supported
3078  */
pci_request_acs(void)3079 void pci_request_acs(void)
3080 {
3081 	pci_acs_enable = 1;
3082 }
3083 
3084 static const char *disable_acs_redir_param;
3085 
3086 /**
3087  * pci_disable_acs_redir - disable ACS redirect capabilities
3088  * @dev: the PCI device
3089  *
3090  * For only devices specified in the disable_acs_redir parameter.
3091  */
pci_disable_acs_redir(struct pci_dev * dev)3092 static void pci_disable_acs_redir(struct pci_dev *dev)
3093 {
3094 	int ret = 0;
3095 	const char *p;
3096 	int pos;
3097 	u16 ctrl;
3098 
3099 	if (!disable_acs_redir_param)
3100 		return;
3101 
3102 	p = disable_acs_redir_param;
3103 	while (*p) {
3104 		ret = pci_dev_str_match(dev, p, &p);
3105 		if (ret < 0) {
3106 			pr_info_once("PCI: Can't parse disable_acs_redir parameter: %s\n",
3107 				     disable_acs_redir_param);
3108 
3109 			break;
3110 		} else if (ret == 1) {
3111 			/* Found a match */
3112 			break;
3113 		}
3114 
3115 		if (*p != ';' && *p != ',') {
3116 			/* End of param or invalid format */
3117 			break;
3118 		}
3119 		p++;
3120 	}
3121 
3122 	if (ret != 1)
3123 		return;
3124 
3125 	if (!pci_dev_specific_disable_acs_redir(dev))
3126 		return;
3127 
3128 	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ACS);
3129 	if (!pos) {
3130 		pci_warn(dev, "cannot disable ACS redirect for this hardware as it does not have ACS capabilities\n");
3131 		return;
3132 	}
3133 
3134 	pci_read_config_word(dev, pos + PCI_ACS_CTRL, &ctrl);
3135 
3136 	/* P2P Request & Completion Redirect */
3137 	ctrl &= ~(PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_EC);
3138 
3139 	pci_write_config_word(dev, pos + PCI_ACS_CTRL, ctrl);
3140 
3141 	pci_info(dev, "disabled ACS redirect\n");
3142 }
3143 
3144 /**
3145  * pci_std_enable_acs - enable ACS on devices using standard ACS capabilites
3146  * @dev: the PCI device
3147  */
pci_std_enable_acs(struct pci_dev * dev)3148 static void pci_std_enable_acs(struct pci_dev *dev)
3149 {
3150 	int pos;
3151 	u16 cap;
3152 	u16 ctrl;
3153 
3154 	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ACS);
3155 	if (!pos)
3156 		return;
3157 
3158 	pci_read_config_word(dev, pos + PCI_ACS_CAP, &cap);
3159 	pci_read_config_word(dev, pos + PCI_ACS_CTRL, &ctrl);
3160 
3161 	/* Source Validation */
3162 	ctrl |= (cap & PCI_ACS_SV);
3163 
3164 	/* P2P Request Redirect */
3165 	ctrl |= (cap & PCI_ACS_RR);
3166 
3167 	/* P2P Completion Redirect */
3168 	ctrl |= (cap & PCI_ACS_CR);
3169 
3170 	/* Upstream Forwarding */
3171 	ctrl |= (cap & PCI_ACS_UF);
3172 
3173 	pci_write_config_word(dev, pos + PCI_ACS_CTRL, ctrl);
3174 }
3175 
3176 /**
3177  * pci_enable_acs - enable ACS if hardware support it
3178  * @dev: the PCI device
3179  */
pci_enable_acs(struct pci_dev * dev)3180 void pci_enable_acs(struct pci_dev *dev)
3181 {
3182 	if (!pci_acs_enable)
3183 		goto disable_acs_redir;
3184 
3185 	if (!pci_dev_specific_enable_acs(dev))
3186 		goto disable_acs_redir;
3187 
3188 	pci_std_enable_acs(dev);
3189 
3190 disable_acs_redir:
3191 	/*
3192 	 * Note: pci_disable_acs_redir() must be called even if ACS was not
3193 	 * enabled by the kernel because it may have been enabled by
3194 	 * platform firmware.  So if we are told to disable it, we should
3195 	 * always disable it after setting the kernel's default
3196 	 * preferences.
3197 	 */
3198 	pci_disable_acs_redir(dev);
3199 }
3200 
pci_acs_flags_enabled(struct pci_dev * pdev,u16 acs_flags)3201 static bool pci_acs_flags_enabled(struct pci_dev *pdev, u16 acs_flags)
3202 {
3203 	int pos;
3204 	u16 cap, ctrl;
3205 
3206 	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ACS);
3207 	if (!pos)
3208 		return false;
3209 
3210 	/*
3211 	 * Except for egress control, capabilities are either required
3212 	 * or only required if controllable.  Features missing from the
3213 	 * capability field can therefore be assumed as hard-wired enabled.
3214 	 */
3215 	pci_read_config_word(pdev, pos + PCI_ACS_CAP, &cap);
3216 	acs_flags &= (cap | PCI_ACS_EC);
3217 
3218 	pci_read_config_word(pdev, pos + PCI_ACS_CTRL, &ctrl);
3219 	return (ctrl & acs_flags) == acs_flags;
3220 }
3221 
3222 /**
3223  * pci_acs_enabled - test ACS against required flags for a given device
3224  * @pdev: device to test
3225  * @acs_flags: required PCI ACS flags
3226  *
3227  * Return true if the device supports the provided flags.  Automatically
3228  * filters out flags that are not implemented on multifunction devices.
3229  *
3230  * Note that this interface checks the effective ACS capabilities of the
3231  * device rather than the actual capabilities.  For instance, most single
3232  * function endpoints are not required to support ACS because they have no
3233  * opportunity for peer-to-peer access.  We therefore return 'true'
3234  * regardless of whether the device exposes an ACS capability.  This makes
3235  * it much easier for callers of this function to ignore the actual type
3236  * or topology of the device when testing ACS support.
3237  */
pci_acs_enabled(struct pci_dev * pdev,u16 acs_flags)3238 bool pci_acs_enabled(struct pci_dev *pdev, u16 acs_flags)
3239 {
3240 	int ret;
3241 
3242 	ret = pci_dev_specific_acs_enabled(pdev, acs_flags);
3243 	if (ret >= 0)
3244 		return ret > 0;
3245 
3246 	/*
3247 	 * Conventional PCI and PCI-X devices never support ACS, either
3248 	 * effectively or actually.  The shared bus topology implies that
3249 	 * any device on the bus can receive or snoop DMA.
3250 	 */
3251 	if (!pci_is_pcie(pdev))
3252 		return false;
3253 
3254 	switch (pci_pcie_type(pdev)) {
3255 	/*
3256 	 * PCI/X-to-PCIe bridges are not specifically mentioned by the spec,
3257 	 * but since their primary interface is PCI/X, we conservatively
3258 	 * handle them as we would a non-PCIe device.
3259 	 */
3260 	case PCI_EXP_TYPE_PCIE_BRIDGE:
3261 	/*
3262 	 * PCIe 3.0, 6.12.1 excludes ACS on these devices.  "ACS is never
3263 	 * applicable... must never implement an ACS Extended Capability...".
3264 	 * This seems arbitrary, but we take a conservative interpretation
3265 	 * of this statement.
3266 	 */
3267 	case PCI_EXP_TYPE_PCI_BRIDGE:
3268 	case PCI_EXP_TYPE_RC_EC:
3269 		return false;
3270 	/*
3271 	 * PCIe 3.0, 6.12.1.1 specifies that downstream and root ports should
3272 	 * implement ACS in order to indicate their peer-to-peer capabilities,
3273 	 * regardless of whether they are single- or multi-function devices.
3274 	 */
3275 	case PCI_EXP_TYPE_DOWNSTREAM:
3276 	case PCI_EXP_TYPE_ROOT_PORT:
3277 		return pci_acs_flags_enabled(pdev, acs_flags);
3278 	/*
3279 	 * PCIe 3.0, 6.12.1.2 specifies ACS capabilities that should be
3280 	 * implemented by the remaining PCIe types to indicate peer-to-peer
3281 	 * capabilities, but only when they are part of a multifunction
3282 	 * device.  The footnote for section 6.12 indicates the specific
3283 	 * PCIe types included here.
3284 	 */
3285 	case PCI_EXP_TYPE_ENDPOINT:
3286 	case PCI_EXP_TYPE_UPSTREAM:
3287 	case PCI_EXP_TYPE_LEG_END:
3288 	case PCI_EXP_TYPE_RC_END:
3289 		if (!pdev->multifunction)
3290 			break;
3291 
3292 		return pci_acs_flags_enabled(pdev, acs_flags);
3293 	}
3294 
3295 	/*
3296 	 * PCIe 3.0, 6.12.1.3 specifies no ACS capabilities are applicable
3297 	 * to single function devices with the exception of downstream ports.
3298 	 */
3299 	return true;
3300 }
3301 
3302 /**
3303  * pci_acs_path_enable - test ACS flags from start to end in a hierarchy
3304  * @start: starting downstream device
3305  * @end: ending upstream device or NULL to search to the root bus
3306  * @acs_flags: required flags
3307  *
3308  * Walk up a device tree from start to end testing PCI ACS support.  If
3309  * any step along the way does not support the required flags, return false.
3310  */
pci_acs_path_enabled(struct pci_dev * start,struct pci_dev * end,u16 acs_flags)3311 bool pci_acs_path_enabled(struct pci_dev *start,
3312 			  struct pci_dev *end, u16 acs_flags)
3313 {
3314 	struct pci_dev *pdev, *parent = start;
3315 
3316 	do {
3317 		pdev = parent;
3318 
3319 		if (!pci_acs_enabled(pdev, acs_flags))
3320 			return false;
3321 
3322 		if (pci_is_root_bus(pdev->bus))
3323 			return (end == NULL);
3324 
3325 		parent = pdev->bus->self;
3326 	} while (pdev != end);
3327 
3328 	return true;
3329 }
3330 
3331 /**
3332  * pci_rebar_find_pos - find position of resize ctrl reg for BAR
3333  * @pdev: PCI device
3334  * @bar: BAR to find
3335  *
3336  * Helper to find the position of the ctrl register for a BAR.
3337  * Returns -ENOTSUPP if resizable BARs are not supported at all.
3338  * Returns -ENOENT if no ctrl register for the BAR could be found.
3339  */
pci_rebar_find_pos(struct pci_dev * pdev,int bar)3340 static int pci_rebar_find_pos(struct pci_dev *pdev, int bar)
3341 {
3342 	unsigned int pos, nbars, i;
3343 	u32 ctrl;
3344 
3345 	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_REBAR);
3346 	if (!pos)
3347 		return -ENOTSUPP;
3348 
3349 	pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
3350 	nbars = (ctrl & PCI_REBAR_CTRL_NBAR_MASK) >>
3351 		    PCI_REBAR_CTRL_NBAR_SHIFT;
3352 
3353 	for (i = 0; i < nbars; i++, pos += 8) {
3354 		int bar_idx;
3355 
3356 		pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
3357 		bar_idx = ctrl & PCI_REBAR_CTRL_BAR_IDX;
3358 		if (bar_idx == bar)
3359 			return pos;
3360 	}
3361 
3362 	return -ENOENT;
3363 }
3364 
3365 /**
3366  * pci_rebar_get_possible_sizes - get possible sizes for BAR
3367  * @pdev: PCI device
3368  * @bar: BAR to query
3369  *
3370  * Get the possible sizes of a resizable BAR as bitmask defined in the spec
3371  * (bit 0=1MB, bit 19=512GB). Returns 0 if BAR isn't resizable.
3372  */
pci_rebar_get_possible_sizes(struct pci_dev * pdev,int bar)3373 u32 pci_rebar_get_possible_sizes(struct pci_dev *pdev, int bar)
3374 {
3375 	int pos;
3376 	u32 cap;
3377 
3378 	pos = pci_rebar_find_pos(pdev, bar);
3379 	if (pos < 0)
3380 		return 0;
3381 
3382 	pci_read_config_dword(pdev, pos + PCI_REBAR_CAP, &cap);
3383 	cap &= PCI_REBAR_CAP_SIZES;
3384 
3385 	/* Sapphire RX 5600 XT Pulse has an invalid cap dword for BAR 0 */
3386 	if (pdev->vendor == PCI_VENDOR_ID_ATI && pdev->device == 0x731f &&
3387 	    bar == 0 && cap == 0x7000)
3388 		cap = 0x3f000;
3389 
3390 	return cap >> 4;
3391 }
3392 
3393 /**
3394  * pci_rebar_get_current_size - get the current size of a BAR
3395  * @pdev: PCI device
3396  * @bar: BAR to set size to
3397  *
3398  * Read the size of a BAR from the resizable BAR config.
3399  * Returns size if found or negative error code.
3400  */
pci_rebar_get_current_size(struct pci_dev * pdev,int bar)3401 int pci_rebar_get_current_size(struct pci_dev *pdev, int bar)
3402 {
3403 	int pos;
3404 	u32 ctrl;
3405 
3406 	pos = pci_rebar_find_pos(pdev, bar);
3407 	if (pos < 0)
3408 		return pos;
3409 
3410 	pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
3411 	return (ctrl & PCI_REBAR_CTRL_BAR_SIZE) >> PCI_REBAR_CTRL_BAR_SHIFT;
3412 }
3413 
3414 /**
3415  * pci_rebar_set_size - set a new size for a BAR
3416  * @pdev: PCI device
3417  * @bar: BAR to set size to
3418  * @size: new size as defined in the spec (0=1MB, 19=512GB)
3419  *
3420  * Set the new size of a BAR as defined in the spec.
3421  * Returns zero if resizing was successful, error code otherwise.
3422  */
pci_rebar_set_size(struct pci_dev * pdev,int bar,int size)3423 int pci_rebar_set_size(struct pci_dev *pdev, int bar, int size)
3424 {
3425 	int pos;
3426 	u32 ctrl;
3427 
3428 	pos = pci_rebar_find_pos(pdev, bar);
3429 	if (pos < 0)
3430 		return pos;
3431 
3432 	pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
3433 	ctrl &= ~PCI_REBAR_CTRL_BAR_SIZE;
3434 	ctrl |= size << PCI_REBAR_CTRL_BAR_SHIFT;
3435 	pci_write_config_dword(pdev, pos + PCI_REBAR_CTRL, ctrl);
3436 	return 0;
3437 }
3438 
3439 /**
3440  * pci_enable_atomic_ops_to_root - enable AtomicOp requests to root port
3441  * @dev: the PCI device
3442  * @cap_mask: mask of desired AtomicOp sizes, including one or more of:
3443  *	PCI_EXP_DEVCAP2_ATOMIC_COMP32
3444  *	PCI_EXP_DEVCAP2_ATOMIC_COMP64
3445  *	PCI_EXP_DEVCAP2_ATOMIC_COMP128
3446  *
3447  * Return 0 if all upstream bridges support AtomicOp routing, egress
3448  * blocking is disabled on all upstream ports, and the root port supports
3449  * the requested completion capabilities (32-bit, 64-bit and/or 128-bit
3450  * AtomicOp completion), or negative otherwise.
3451  */
pci_enable_atomic_ops_to_root(struct pci_dev * dev,u32 cap_mask)3452 int pci_enable_atomic_ops_to_root(struct pci_dev *dev, u32 cap_mask)
3453 {
3454 	struct pci_bus *bus = dev->bus;
3455 	struct pci_dev *bridge;
3456 	u32 cap, ctl2;
3457 
3458 	if (!pci_is_pcie(dev))
3459 		return -EINVAL;
3460 
3461 	/*
3462 	 * Per PCIe r4.0, sec 6.15, endpoints and root ports may be
3463 	 * AtomicOp requesters.  For now, we only support endpoints as
3464 	 * requesters and root ports as completers.  No endpoints as
3465 	 * completers, and no peer-to-peer.
3466 	 */
3467 
3468 	switch (pci_pcie_type(dev)) {
3469 	case PCI_EXP_TYPE_ENDPOINT:
3470 	case PCI_EXP_TYPE_LEG_END:
3471 	case PCI_EXP_TYPE_RC_END:
3472 		break;
3473 	default:
3474 		return -EINVAL;
3475 	}
3476 
3477 	while (bus->parent) {
3478 		bridge = bus->self;
3479 
3480 		pcie_capability_read_dword(bridge, PCI_EXP_DEVCAP2, &cap);
3481 
3482 		switch (pci_pcie_type(bridge)) {
3483 		/* Ensure switch ports support AtomicOp routing */
3484 		case PCI_EXP_TYPE_UPSTREAM:
3485 		case PCI_EXP_TYPE_DOWNSTREAM:
3486 			if (!(cap & PCI_EXP_DEVCAP2_ATOMIC_ROUTE))
3487 				return -EINVAL;
3488 			break;
3489 
3490 		/* Ensure root port supports all the sizes we care about */
3491 		case PCI_EXP_TYPE_ROOT_PORT:
3492 			if ((cap & cap_mask) != cap_mask)
3493 				return -EINVAL;
3494 			break;
3495 		}
3496 
3497 		/* Ensure upstream ports don't block AtomicOps on egress */
3498 		if (!bridge->has_secondary_link) {
3499 			pcie_capability_read_dword(bridge, PCI_EXP_DEVCTL2,
3500 						   &ctl2);
3501 			if (ctl2 & PCI_EXP_DEVCTL2_ATOMIC_EGRESS_BLOCK)
3502 				return -EINVAL;
3503 		}
3504 
3505 		bus = bus->parent;
3506 	}
3507 
3508 	pcie_capability_set_word(dev, PCI_EXP_DEVCTL2,
3509 				 PCI_EXP_DEVCTL2_ATOMIC_REQ);
3510 	return 0;
3511 }
3512 EXPORT_SYMBOL(pci_enable_atomic_ops_to_root);
3513 
3514 /**
3515  * pci_swizzle_interrupt_pin - swizzle INTx for device behind bridge
3516  * @dev: the PCI device
3517  * @pin: the INTx pin (1=INTA, 2=INTB, 3=INTC, 4=INTD)
3518  *
3519  * Perform INTx swizzling for a device behind one level of bridge.  This is
3520  * required by section 9.1 of the PCI-to-PCI bridge specification for devices
3521  * behind bridges on add-in cards.  For devices with ARI enabled, the slot
3522  * number is always 0 (see the Implementation Note in section 2.2.8.1 of
3523  * the PCI Express Base Specification, Revision 2.1)
3524  */
pci_swizzle_interrupt_pin(const struct pci_dev * dev,u8 pin)3525 u8 pci_swizzle_interrupt_pin(const struct pci_dev *dev, u8 pin)
3526 {
3527 	int slot;
3528 
3529 	if (pci_ari_enabled(dev->bus))
3530 		slot = 0;
3531 	else
3532 		slot = PCI_SLOT(dev->devfn);
3533 
3534 	return (((pin - 1) + slot) % 4) + 1;
3535 }
3536 
pci_get_interrupt_pin(struct pci_dev * dev,struct pci_dev ** bridge)3537 int pci_get_interrupt_pin(struct pci_dev *dev, struct pci_dev **bridge)
3538 {
3539 	u8 pin;
3540 
3541 	pin = dev->pin;
3542 	if (!pin)
3543 		return -1;
3544 
3545 	while (!pci_is_root_bus(dev->bus)) {
3546 		pin = pci_swizzle_interrupt_pin(dev, pin);
3547 		dev = dev->bus->self;
3548 	}
3549 	*bridge = dev;
3550 	return pin;
3551 }
3552 
3553 /**
3554  * pci_common_swizzle - swizzle INTx all the way to root bridge
3555  * @dev: the PCI device
3556  * @pinp: pointer to the INTx pin value (1=INTA, 2=INTB, 3=INTD, 4=INTD)
3557  *
3558  * Perform INTx swizzling for a device.  This traverses through all PCI-to-PCI
3559  * bridges all the way up to a PCI root bus.
3560  */
pci_common_swizzle(struct pci_dev * dev,u8 * pinp)3561 u8 pci_common_swizzle(struct pci_dev *dev, u8 *pinp)
3562 {
3563 	u8 pin = *pinp;
3564 
3565 	while (!pci_is_root_bus(dev->bus)) {
3566 		pin = pci_swizzle_interrupt_pin(dev, pin);
3567 		dev = dev->bus->self;
3568 	}
3569 	*pinp = pin;
3570 	return PCI_SLOT(dev->devfn);
3571 }
3572 EXPORT_SYMBOL_GPL(pci_common_swizzle);
3573 
3574 /**
3575  *	pci_release_region - Release a PCI bar
3576  *	@pdev: PCI device whose resources were previously reserved by pci_request_region
3577  *	@bar: BAR to release
3578  *
3579  *	Releases the PCI I/O and memory resources previously reserved by a
3580  *	successful call to pci_request_region.  Call this function only
3581  *	after all use of the PCI regions has ceased.
3582  */
pci_release_region(struct pci_dev * pdev,int bar)3583 void pci_release_region(struct pci_dev *pdev, int bar)
3584 {
3585 	struct pci_devres *dr;
3586 
3587 	if (pci_resource_len(pdev, bar) == 0)
3588 		return;
3589 	if (pci_resource_flags(pdev, bar) & IORESOURCE_IO)
3590 		release_region(pci_resource_start(pdev, bar),
3591 				pci_resource_len(pdev, bar));
3592 	else if (pci_resource_flags(pdev, bar) & IORESOURCE_MEM)
3593 		release_mem_region(pci_resource_start(pdev, bar),
3594 				pci_resource_len(pdev, bar));
3595 
3596 	dr = find_pci_dr(pdev);
3597 	if (dr)
3598 		dr->region_mask &= ~(1 << bar);
3599 }
3600 EXPORT_SYMBOL(pci_release_region);
3601 
3602 /**
3603  *	__pci_request_region - Reserved PCI I/O and memory resource
3604  *	@pdev: PCI device whose resources are to be reserved
3605  *	@bar: BAR to be reserved
3606  *	@res_name: Name to be associated with resource.
3607  *	@exclusive: whether the region access is exclusive or not
3608  *
3609  *	Mark the PCI region associated with PCI device @pdev BR @bar as
3610  *	being reserved by owner @res_name.  Do not access any
3611  *	address inside the PCI regions unless this call returns
3612  *	successfully.
3613  *
3614  *	If @exclusive is set, then the region is marked so that userspace
3615  *	is explicitly not allowed to map the resource via /dev/mem or
3616  *	sysfs MMIO access.
3617  *
3618  *	Returns 0 on success, or %EBUSY on error.  A warning
3619  *	message is also printed on failure.
3620  */
__pci_request_region(struct pci_dev * pdev,int bar,const char * res_name,int exclusive)3621 static int __pci_request_region(struct pci_dev *pdev, int bar,
3622 				const char *res_name, int exclusive)
3623 {
3624 	struct pci_devres *dr;
3625 
3626 	if (pci_resource_len(pdev, bar) == 0)
3627 		return 0;
3628 
3629 	if (pci_resource_flags(pdev, bar) & IORESOURCE_IO) {
3630 		if (!request_region(pci_resource_start(pdev, bar),
3631 			    pci_resource_len(pdev, bar), res_name))
3632 			goto err_out;
3633 	} else if (pci_resource_flags(pdev, bar) & IORESOURCE_MEM) {
3634 		if (!__request_mem_region(pci_resource_start(pdev, bar),
3635 					pci_resource_len(pdev, bar), res_name,
3636 					exclusive))
3637 			goto err_out;
3638 	}
3639 
3640 	dr = find_pci_dr(pdev);
3641 	if (dr)
3642 		dr->region_mask |= 1 << bar;
3643 
3644 	return 0;
3645 
3646 err_out:
3647 	pci_warn(pdev, "BAR %d: can't reserve %pR\n", bar,
3648 		 &pdev->resource[bar]);
3649 	return -EBUSY;
3650 }
3651 
3652 /**
3653  *	pci_request_region - Reserve PCI I/O and memory resource
3654  *	@pdev: PCI device whose resources are to be reserved
3655  *	@bar: BAR to be reserved
3656  *	@res_name: Name to be associated with resource
3657  *
3658  *	Mark the PCI region associated with PCI device @pdev BAR @bar as
3659  *	being reserved by owner @res_name.  Do not access any
3660  *	address inside the PCI regions unless this call returns
3661  *	successfully.
3662  *
3663  *	Returns 0 on success, or %EBUSY on error.  A warning
3664  *	message is also printed on failure.
3665  */
pci_request_region(struct pci_dev * pdev,int bar,const char * res_name)3666 int pci_request_region(struct pci_dev *pdev, int bar, const char *res_name)
3667 {
3668 	return __pci_request_region(pdev, bar, res_name, 0);
3669 }
3670 EXPORT_SYMBOL(pci_request_region);
3671 
3672 /**
3673  *	pci_request_region_exclusive - Reserved PCI I/O and memory resource
3674  *	@pdev: PCI device whose resources are to be reserved
3675  *	@bar: BAR to be reserved
3676  *	@res_name: Name to be associated with resource.
3677  *
3678  *	Mark the PCI region associated with PCI device @pdev BR @bar as
3679  *	being reserved by owner @res_name.  Do not access any
3680  *	address inside the PCI regions unless this call returns
3681  *	successfully.
3682  *
3683  *	Returns 0 on success, or %EBUSY on error.  A warning
3684  *	message is also printed on failure.
3685  *
3686  *	The key difference that _exclusive makes it that userspace is
3687  *	explicitly not allowed to map the resource via /dev/mem or
3688  *	sysfs.
3689  */
pci_request_region_exclusive(struct pci_dev * pdev,int bar,const char * res_name)3690 int pci_request_region_exclusive(struct pci_dev *pdev, int bar,
3691 				 const char *res_name)
3692 {
3693 	return __pci_request_region(pdev, bar, res_name, IORESOURCE_EXCLUSIVE);
3694 }
3695 EXPORT_SYMBOL(pci_request_region_exclusive);
3696 
3697 /**
3698  * pci_release_selected_regions - Release selected PCI I/O and memory resources
3699  * @pdev: PCI device whose resources were previously reserved
3700  * @bars: Bitmask of BARs to be released
3701  *
3702  * Release selected PCI I/O and memory resources previously reserved.
3703  * Call this function only after all use of the PCI regions has ceased.
3704  */
pci_release_selected_regions(struct pci_dev * pdev,int bars)3705 void pci_release_selected_regions(struct pci_dev *pdev, int bars)
3706 {
3707 	int i;
3708 
3709 	for (i = 0; i < 6; i++)
3710 		if (bars & (1 << i))
3711 			pci_release_region(pdev, i);
3712 }
3713 EXPORT_SYMBOL(pci_release_selected_regions);
3714 
__pci_request_selected_regions(struct pci_dev * pdev,int bars,const char * res_name,int excl)3715 static int __pci_request_selected_regions(struct pci_dev *pdev, int bars,
3716 					  const char *res_name, int excl)
3717 {
3718 	int i;
3719 
3720 	for (i = 0; i < 6; i++)
3721 		if (bars & (1 << i))
3722 			if (__pci_request_region(pdev, i, res_name, excl))
3723 				goto err_out;
3724 	return 0;
3725 
3726 err_out:
3727 	while (--i >= 0)
3728 		if (bars & (1 << i))
3729 			pci_release_region(pdev, i);
3730 
3731 	return -EBUSY;
3732 }
3733 
3734 
3735 /**
3736  * pci_request_selected_regions - Reserve selected PCI I/O and memory resources
3737  * @pdev: PCI device whose resources are to be reserved
3738  * @bars: Bitmask of BARs to be requested
3739  * @res_name: Name to be associated with resource
3740  */
pci_request_selected_regions(struct pci_dev * pdev,int bars,const char * res_name)3741 int pci_request_selected_regions(struct pci_dev *pdev, int bars,
3742 				 const char *res_name)
3743 {
3744 	return __pci_request_selected_regions(pdev, bars, res_name, 0);
3745 }
3746 EXPORT_SYMBOL(pci_request_selected_regions);
3747 
pci_request_selected_regions_exclusive(struct pci_dev * pdev,int bars,const char * res_name)3748 int pci_request_selected_regions_exclusive(struct pci_dev *pdev, int bars,
3749 					   const char *res_name)
3750 {
3751 	return __pci_request_selected_regions(pdev, bars, res_name,
3752 			IORESOURCE_EXCLUSIVE);
3753 }
3754 EXPORT_SYMBOL(pci_request_selected_regions_exclusive);
3755 
3756 /**
3757  *	pci_release_regions - Release reserved PCI I/O and memory resources
3758  *	@pdev: PCI device whose resources were previously reserved by pci_request_regions
3759  *
3760  *	Releases all PCI I/O and memory resources previously reserved by a
3761  *	successful call to pci_request_regions.  Call this function only
3762  *	after all use of the PCI regions has ceased.
3763  */
3764 
pci_release_regions(struct pci_dev * pdev)3765 void pci_release_regions(struct pci_dev *pdev)
3766 {
3767 	pci_release_selected_regions(pdev, (1 << 6) - 1);
3768 }
3769 EXPORT_SYMBOL(pci_release_regions);
3770 
3771 /**
3772  *	pci_request_regions - Reserved PCI I/O and memory resources
3773  *	@pdev: PCI device whose resources are to be reserved
3774  *	@res_name: Name to be associated with resource.
3775  *
3776  *	Mark all PCI regions associated with PCI device @pdev as
3777  *	being reserved by owner @res_name.  Do not access any
3778  *	address inside the PCI regions unless this call returns
3779  *	successfully.
3780  *
3781  *	Returns 0 on success, or %EBUSY on error.  A warning
3782  *	message is also printed on failure.
3783  */
pci_request_regions(struct pci_dev * pdev,const char * res_name)3784 int pci_request_regions(struct pci_dev *pdev, const char *res_name)
3785 {
3786 	return pci_request_selected_regions(pdev, ((1 << 6) - 1), res_name);
3787 }
3788 EXPORT_SYMBOL(pci_request_regions);
3789 
3790 /**
3791  *	pci_request_regions_exclusive - Reserved PCI I/O and memory resources
3792  *	@pdev: PCI device whose resources are to be reserved
3793  *	@res_name: Name to be associated with resource.
3794  *
3795  *	Mark all PCI regions associated with PCI device @pdev as
3796  *	being reserved by owner @res_name.  Do not access any
3797  *	address inside the PCI regions unless this call returns
3798  *	successfully.
3799  *
3800  *	pci_request_regions_exclusive() will mark the region so that
3801  *	/dev/mem and the sysfs MMIO access will not be allowed.
3802  *
3803  *	Returns 0 on success, or %EBUSY on error.  A warning
3804  *	message is also printed on failure.
3805  */
pci_request_regions_exclusive(struct pci_dev * pdev,const char * res_name)3806 int pci_request_regions_exclusive(struct pci_dev *pdev, const char *res_name)
3807 {
3808 	return pci_request_selected_regions_exclusive(pdev,
3809 					((1 << 6) - 1), res_name);
3810 }
3811 EXPORT_SYMBOL(pci_request_regions_exclusive);
3812 
3813 /*
3814  * Record the PCI IO range (expressed as CPU physical address + size).
3815  * Return a negative value if an error has occured, zero otherwise
3816  */
pci_register_io_range(struct fwnode_handle * fwnode,phys_addr_t addr,resource_size_t size)3817 int pci_register_io_range(struct fwnode_handle *fwnode, phys_addr_t addr,
3818 			resource_size_t	size)
3819 {
3820 	int ret = 0;
3821 #ifdef PCI_IOBASE
3822 	struct logic_pio_hwaddr *range;
3823 
3824 	if (!size || addr + size < addr)
3825 		return -EINVAL;
3826 
3827 	range = kzalloc(sizeof(*range), GFP_ATOMIC);
3828 	if (!range)
3829 		return -ENOMEM;
3830 
3831 	range->fwnode = fwnode;
3832 	range->size = size;
3833 	range->hw_start = addr;
3834 	range->flags = LOGIC_PIO_CPU_MMIO;
3835 
3836 	ret = logic_pio_register_range(range);
3837 	if (ret)
3838 		kfree(range);
3839 
3840 	/* Ignore duplicates due to deferred probing */
3841 	if (ret == -EEXIST)
3842 		ret = 0;
3843 #endif
3844 
3845 	return ret;
3846 }
3847 
pci_pio_to_address(unsigned long pio)3848 phys_addr_t pci_pio_to_address(unsigned long pio)
3849 {
3850 	phys_addr_t address = (phys_addr_t)OF_BAD_ADDR;
3851 
3852 #ifdef PCI_IOBASE
3853 	if (pio >= MMIO_UPPER_LIMIT)
3854 		return address;
3855 
3856 	address = logic_pio_to_hwaddr(pio);
3857 #endif
3858 
3859 	return address;
3860 }
3861 
pci_address_to_pio(phys_addr_t address)3862 unsigned long __weak pci_address_to_pio(phys_addr_t address)
3863 {
3864 #ifdef PCI_IOBASE
3865 	return logic_pio_trans_cpuaddr(address);
3866 #else
3867 	if (address > IO_SPACE_LIMIT)
3868 		return (unsigned long)-1;
3869 
3870 	return (unsigned long) address;
3871 #endif
3872 }
3873 
3874 /**
3875  *	pci_remap_iospace - Remap the memory mapped I/O space
3876  *	@res: Resource describing the I/O space
3877  *	@phys_addr: physical address of range to be mapped
3878  *
3879  *	Remap the memory mapped I/O space described by the @res
3880  *	and the CPU physical address @phys_addr into virtual address space.
3881  *	Only architectures that have memory mapped IO functions defined
3882  *	(and the PCI_IOBASE value defined) should call this function.
3883  */
pci_remap_iospace(const struct resource * res,phys_addr_t phys_addr)3884 int pci_remap_iospace(const struct resource *res, phys_addr_t phys_addr)
3885 {
3886 #if defined(PCI_IOBASE) && defined(CONFIG_MMU)
3887 	unsigned long vaddr = (unsigned long)PCI_IOBASE + res->start;
3888 
3889 	if (!(res->flags & IORESOURCE_IO))
3890 		return -EINVAL;
3891 
3892 	if (res->end > IO_SPACE_LIMIT)
3893 		return -EINVAL;
3894 
3895 	return ioremap_page_range(vaddr, vaddr + resource_size(res), phys_addr,
3896 				  pgprot_device(PAGE_KERNEL));
3897 #else
3898 	/* this architecture does not have memory mapped I/O space,
3899 	   so this function should never be called */
3900 	WARN_ONCE(1, "This architecture does not support memory mapped I/O\n");
3901 	return -ENODEV;
3902 #endif
3903 }
3904 EXPORT_SYMBOL(pci_remap_iospace);
3905 
3906 /**
3907  *	pci_unmap_iospace - Unmap the memory mapped I/O space
3908  *	@res: resource to be unmapped
3909  *
3910  *	Unmap the CPU virtual address @res from virtual address space.
3911  *	Only architectures that have memory mapped IO functions defined
3912  *	(and the PCI_IOBASE value defined) should call this function.
3913  */
pci_unmap_iospace(struct resource * res)3914 void pci_unmap_iospace(struct resource *res)
3915 {
3916 #if defined(PCI_IOBASE) && defined(CONFIG_MMU)
3917 	unsigned long vaddr = (unsigned long)PCI_IOBASE + res->start;
3918 
3919 	unmap_kernel_range(vaddr, resource_size(res));
3920 #endif
3921 }
3922 EXPORT_SYMBOL(pci_unmap_iospace);
3923 
devm_pci_unmap_iospace(struct device * dev,void * ptr)3924 static void devm_pci_unmap_iospace(struct device *dev, void *ptr)
3925 {
3926 	struct resource **res = ptr;
3927 
3928 	pci_unmap_iospace(*res);
3929 }
3930 
3931 /**
3932  * devm_pci_remap_iospace - Managed pci_remap_iospace()
3933  * @dev: Generic device to remap IO address for
3934  * @res: Resource describing the I/O space
3935  * @phys_addr: physical address of range to be mapped
3936  *
3937  * Managed pci_remap_iospace().  Map is automatically unmapped on driver
3938  * detach.
3939  */
devm_pci_remap_iospace(struct device * dev,const struct resource * res,phys_addr_t phys_addr)3940 int devm_pci_remap_iospace(struct device *dev, const struct resource *res,
3941 			   phys_addr_t phys_addr)
3942 {
3943 	const struct resource **ptr;
3944 	int error;
3945 
3946 	ptr = devres_alloc(devm_pci_unmap_iospace, sizeof(*ptr), GFP_KERNEL);
3947 	if (!ptr)
3948 		return -ENOMEM;
3949 
3950 	error = pci_remap_iospace(res, phys_addr);
3951 	if (error) {
3952 		devres_free(ptr);
3953 	} else	{
3954 		*ptr = res;
3955 		devres_add(dev, ptr);
3956 	}
3957 
3958 	return error;
3959 }
3960 EXPORT_SYMBOL(devm_pci_remap_iospace);
3961 
3962 /**
3963  * devm_pci_remap_cfgspace - Managed pci_remap_cfgspace()
3964  * @dev: Generic device to remap IO address for
3965  * @offset: Resource address to map
3966  * @size: Size of map
3967  *
3968  * Managed pci_remap_cfgspace().  Map is automatically unmapped on driver
3969  * detach.
3970  */
devm_pci_remap_cfgspace(struct device * dev,resource_size_t offset,resource_size_t size)3971 void __iomem *devm_pci_remap_cfgspace(struct device *dev,
3972 				      resource_size_t offset,
3973 				      resource_size_t size)
3974 {
3975 	void __iomem **ptr, *addr;
3976 
3977 	ptr = devres_alloc(devm_ioremap_release, sizeof(*ptr), GFP_KERNEL);
3978 	if (!ptr)
3979 		return NULL;
3980 
3981 	addr = pci_remap_cfgspace(offset, size);
3982 	if (addr) {
3983 		*ptr = addr;
3984 		devres_add(dev, ptr);
3985 	} else
3986 		devres_free(ptr);
3987 
3988 	return addr;
3989 }
3990 EXPORT_SYMBOL(devm_pci_remap_cfgspace);
3991 
3992 /**
3993  * devm_pci_remap_cfg_resource - check, request region and ioremap cfg resource
3994  * @dev: generic device to handle the resource for
3995  * @res: configuration space resource to be handled
3996  *
3997  * Checks that a resource is a valid memory region, requests the memory
3998  * region and ioremaps with pci_remap_cfgspace() API that ensures the
3999  * proper PCI configuration space memory attributes are guaranteed.
4000  *
4001  * All operations are managed and will be undone on driver detach.
4002  *
4003  * Returns a pointer to the remapped memory or an ERR_PTR() encoded error code
4004  * on failure. Usage example::
4005  *
4006  *	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
4007  *	base = devm_pci_remap_cfg_resource(&pdev->dev, res);
4008  *	if (IS_ERR(base))
4009  *		return PTR_ERR(base);
4010  */
devm_pci_remap_cfg_resource(struct device * dev,struct resource * res)4011 void __iomem *devm_pci_remap_cfg_resource(struct device *dev,
4012 					  struct resource *res)
4013 {
4014 	resource_size_t size;
4015 	const char *name;
4016 	void __iomem *dest_ptr;
4017 
4018 	BUG_ON(!dev);
4019 
4020 	if (!res || resource_type(res) != IORESOURCE_MEM) {
4021 		dev_err(dev, "invalid resource\n");
4022 		return IOMEM_ERR_PTR(-EINVAL);
4023 	}
4024 
4025 	size = resource_size(res);
4026 	name = res->name ?: dev_name(dev);
4027 
4028 	if (!devm_request_mem_region(dev, res->start, size, name)) {
4029 		dev_err(dev, "can't request region for resource %pR\n", res);
4030 		return IOMEM_ERR_PTR(-EBUSY);
4031 	}
4032 
4033 	dest_ptr = devm_pci_remap_cfgspace(dev, res->start, size);
4034 	if (!dest_ptr) {
4035 		dev_err(dev, "ioremap failed for resource %pR\n", res);
4036 		devm_release_mem_region(dev, res->start, size);
4037 		dest_ptr = IOMEM_ERR_PTR(-ENOMEM);
4038 	}
4039 
4040 	return dest_ptr;
4041 }
4042 EXPORT_SYMBOL(devm_pci_remap_cfg_resource);
4043 
__pci_set_master(struct pci_dev * dev,bool enable)4044 static void __pci_set_master(struct pci_dev *dev, bool enable)
4045 {
4046 	u16 old_cmd, cmd;
4047 
4048 	pci_read_config_word(dev, PCI_COMMAND, &old_cmd);
4049 	if (enable)
4050 		cmd = old_cmd | PCI_COMMAND_MASTER;
4051 	else
4052 		cmd = old_cmd & ~PCI_COMMAND_MASTER;
4053 	if (cmd != old_cmd) {
4054 		pci_dbg(dev, "%s bus mastering\n",
4055 			enable ? "enabling" : "disabling");
4056 		pci_write_config_word(dev, PCI_COMMAND, cmd);
4057 	}
4058 	dev->is_busmaster = enable;
4059 }
4060 
4061 /**
4062  * pcibios_setup - process "pci=" kernel boot arguments
4063  * @str: string used to pass in "pci=" kernel boot arguments
4064  *
4065  * Process kernel boot arguments.  This is the default implementation.
4066  * Architecture specific implementations can override this as necessary.
4067  */
pcibios_setup(char * str)4068 char * __weak __init pcibios_setup(char *str)
4069 {
4070 	return str;
4071 }
4072 
4073 /**
4074  * pcibios_set_master - enable PCI bus-mastering for device dev
4075  * @dev: the PCI device to enable
4076  *
4077  * Enables PCI bus-mastering for the device.  This is the default
4078  * implementation.  Architecture specific implementations can override
4079  * this if necessary.
4080  */
pcibios_set_master(struct pci_dev * dev)4081 void __weak pcibios_set_master(struct pci_dev *dev)
4082 {
4083 	u8 lat;
4084 
4085 	/* The latency timer doesn't apply to PCIe (either Type 0 or Type 1) */
4086 	if (pci_is_pcie(dev))
4087 		return;
4088 
4089 	pci_read_config_byte(dev, PCI_LATENCY_TIMER, &lat);
4090 	if (lat < 16)
4091 		lat = (64 <= pcibios_max_latency) ? 64 : pcibios_max_latency;
4092 	else if (lat > pcibios_max_latency)
4093 		lat = pcibios_max_latency;
4094 	else
4095 		return;
4096 
4097 	pci_write_config_byte(dev, PCI_LATENCY_TIMER, lat);
4098 }
4099 
4100 /**
4101  * pci_set_master - enables bus-mastering for device dev
4102  * @dev: the PCI device to enable
4103  *
4104  * Enables bus-mastering on the device and calls pcibios_set_master()
4105  * to do the needed arch specific settings.
4106  */
pci_set_master(struct pci_dev * dev)4107 void pci_set_master(struct pci_dev *dev)
4108 {
4109 	__pci_set_master(dev, true);
4110 	pcibios_set_master(dev);
4111 }
4112 EXPORT_SYMBOL(pci_set_master);
4113 
4114 /**
4115  * pci_clear_master - disables bus-mastering for device dev
4116  * @dev: the PCI device to disable
4117  */
pci_clear_master(struct pci_dev * dev)4118 void pci_clear_master(struct pci_dev *dev)
4119 {
4120 	__pci_set_master(dev, false);
4121 }
4122 EXPORT_SYMBOL(pci_clear_master);
4123 
4124 /**
4125  * pci_set_cacheline_size - ensure the CACHE_LINE_SIZE register is programmed
4126  * @dev: the PCI device for which MWI is to be enabled
4127  *
4128  * Helper function for pci_set_mwi.
4129  * Originally copied from drivers/net/acenic.c.
4130  * Copyright 1998-2001 by Jes Sorensen, <jes@trained-monkey.org>.
4131  *
4132  * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
4133  */
pci_set_cacheline_size(struct pci_dev * dev)4134 int pci_set_cacheline_size(struct pci_dev *dev)
4135 {
4136 	u8 cacheline_size;
4137 
4138 	if (!pci_cache_line_size)
4139 		return -EINVAL;
4140 
4141 	/* Validate current setting: the PCI_CACHE_LINE_SIZE must be
4142 	   equal to or multiple of the right value. */
4143 	pci_read_config_byte(dev, PCI_CACHE_LINE_SIZE, &cacheline_size);
4144 	if (cacheline_size >= pci_cache_line_size &&
4145 	    (cacheline_size % pci_cache_line_size) == 0)
4146 		return 0;
4147 
4148 	/* Write the correct value. */
4149 	pci_write_config_byte(dev, PCI_CACHE_LINE_SIZE, pci_cache_line_size);
4150 	/* Read it back. */
4151 	pci_read_config_byte(dev, PCI_CACHE_LINE_SIZE, &cacheline_size);
4152 	if (cacheline_size == pci_cache_line_size)
4153 		return 0;
4154 
4155 	pci_printk(KERN_DEBUG, dev, "cache line size of %d is not supported\n",
4156 		   pci_cache_line_size << 2);
4157 
4158 	return -EINVAL;
4159 }
4160 EXPORT_SYMBOL_GPL(pci_set_cacheline_size);
4161 
4162 /**
4163  * pci_set_mwi - enables memory-write-invalidate PCI transaction
4164  * @dev: the PCI device for which MWI is enabled
4165  *
4166  * Enables the Memory-Write-Invalidate transaction in %PCI_COMMAND.
4167  *
4168  * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
4169  */
pci_set_mwi(struct pci_dev * dev)4170 int pci_set_mwi(struct pci_dev *dev)
4171 {
4172 #ifdef PCI_DISABLE_MWI
4173 	return 0;
4174 #else
4175 	int rc;
4176 	u16 cmd;
4177 
4178 	rc = pci_set_cacheline_size(dev);
4179 	if (rc)
4180 		return rc;
4181 
4182 	pci_read_config_word(dev, PCI_COMMAND, &cmd);
4183 	if (!(cmd & PCI_COMMAND_INVALIDATE)) {
4184 		pci_dbg(dev, "enabling Mem-Wr-Inval\n");
4185 		cmd |= PCI_COMMAND_INVALIDATE;
4186 		pci_write_config_word(dev, PCI_COMMAND, cmd);
4187 	}
4188 	return 0;
4189 #endif
4190 }
4191 EXPORT_SYMBOL(pci_set_mwi);
4192 
4193 /**
4194  * pcim_set_mwi - a device-managed pci_set_mwi()
4195  * @dev: the PCI device for which MWI is enabled
4196  *
4197  * Managed pci_set_mwi().
4198  *
4199  * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
4200  */
pcim_set_mwi(struct pci_dev * dev)4201 int pcim_set_mwi(struct pci_dev *dev)
4202 {
4203 	struct pci_devres *dr;
4204 
4205 	dr = find_pci_dr(dev);
4206 	if (!dr)
4207 		return -ENOMEM;
4208 
4209 	dr->mwi = 1;
4210 	return pci_set_mwi(dev);
4211 }
4212 EXPORT_SYMBOL(pcim_set_mwi);
4213 
4214 /**
4215  * pci_try_set_mwi - enables memory-write-invalidate PCI transaction
4216  * @dev: the PCI device for which MWI is enabled
4217  *
4218  * Enables the Memory-Write-Invalidate transaction in %PCI_COMMAND.
4219  * Callers are not required to check the return value.
4220  *
4221  * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
4222  */
pci_try_set_mwi(struct pci_dev * dev)4223 int pci_try_set_mwi(struct pci_dev *dev)
4224 {
4225 #ifdef PCI_DISABLE_MWI
4226 	return 0;
4227 #else
4228 	return pci_set_mwi(dev);
4229 #endif
4230 }
4231 EXPORT_SYMBOL(pci_try_set_mwi);
4232 
4233 /**
4234  * pci_clear_mwi - disables Memory-Write-Invalidate for device dev
4235  * @dev: the PCI device to disable
4236  *
4237  * Disables PCI Memory-Write-Invalidate transaction on the device
4238  */
pci_clear_mwi(struct pci_dev * dev)4239 void pci_clear_mwi(struct pci_dev *dev)
4240 {
4241 #ifndef PCI_DISABLE_MWI
4242 	u16 cmd;
4243 
4244 	pci_read_config_word(dev, PCI_COMMAND, &cmd);
4245 	if (cmd & PCI_COMMAND_INVALIDATE) {
4246 		cmd &= ~PCI_COMMAND_INVALIDATE;
4247 		pci_write_config_word(dev, PCI_COMMAND, cmd);
4248 	}
4249 #endif
4250 }
4251 EXPORT_SYMBOL(pci_clear_mwi);
4252 
4253 /**
4254  * pci_intx - enables/disables PCI INTx for device dev
4255  * @pdev: the PCI device to operate on
4256  * @enable: boolean: whether to enable or disable PCI INTx
4257  *
4258  * Enables/disables PCI INTx for device dev
4259  */
pci_intx(struct pci_dev * pdev,int enable)4260 void pci_intx(struct pci_dev *pdev, int enable)
4261 {
4262 	u16 pci_command, new;
4263 
4264 	pci_read_config_word(pdev, PCI_COMMAND, &pci_command);
4265 
4266 	if (enable)
4267 		new = pci_command & ~PCI_COMMAND_INTX_DISABLE;
4268 	else
4269 		new = pci_command | PCI_COMMAND_INTX_DISABLE;
4270 
4271 	if (new != pci_command) {
4272 		struct pci_devres *dr;
4273 
4274 		pci_write_config_word(pdev, PCI_COMMAND, new);
4275 
4276 		dr = find_pci_dr(pdev);
4277 		if (dr && !dr->restore_intx) {
4278 			dr->restore_intx = 1;
4279 			dr->orig_intx = !enable;
4280 		}
4281 	}
4282 }
4283 EXPORT_SYMBOL_GPL(pci_intx);
4284 
pci_check_and_set_intx_mask(struct pci_dev * dev,bool mask)4285 static bool pci_check_and_set_intx_mask(struct pci_dev *dev, bool mask)
4286 {
4287 	struct pci_bus *bus = dev->bus;
4288 	bool mask_updated = true;
4289 	u32 cmd_status_dword;
4290 	u16 origcmd, newcmd;
4291 	unsigned long flags;
4292 	bool irq_pending;
4293 
4294 	/*
4295 	 * We do a single dword read to retrieve both command and status.
4296 	 * Document assumptions that make this possible.
4297 	 */
4298 	BUILD_BUG_ON(PCI_COMMAND % 4);
4299 	BUILD_BUG_ON(PCI_COMMAND + 2 != PCI_STATUS);
4300 
4301 	raw_spin_lock_irqsave(&pci_lock, flags);
4302 
4303 	bus->ops->read(bus, dev->devfn, PCI_COMMAND, 4, &cmd_status_dword);
4304 
4305 	irq_pending = (cmd_status_dword >> 16) & PCI_STATUS_INTERRUPT;
4306 
4307 	/*
4308 	 * Check interrupt status register to see whether our device
4309 	 * triggered the interrupt (when masking) or the next IRQ is
4310 	 * already pending (when unmasking).
4311 	 */
4312 	if (mask != irq_pending) {
4313 		mask_updated = false;
4314 		goto done;
4315 	}
4316 
4317 	origcmd = cmd_status_dword;
4318 	newcmd = origcmd & ~PCI_COMMAND_INTX_DISABLE;
4319 	if (mask)
4320 		newcmd |= PCI_COMMAND_INTX_DISABLE;
4321 	if (newcmd != origcmd)
4322 		bus->ops->write(bus, dev->devfn, PCI_COMMAND, 2, newcmd);
4323 
4324 done:
4325 	raw_spin_unlock_irqrestore(&pci_lock, flags);
4326 
4327 	return mask_updated;
4328 }
4329 
4330 /**
4331  * pci_check_and_mask_intx - mask INTx on pending interrupt
4332  * @dev: the PCI device to operate on
4333  *
4334  * Check if the device dev has its INTx line asserted, mask it and
4335  * return true in that case. False is returned if no interrupt was
4336  * pending.
4337  */
pci_check_and_mask_intx(struct pci_dev * dev)4338 bool pci_check_and_mask_intx(struct pci_dev *dev)
4339 {
4340 	return pci_check_and_set_intx_mask(dev, true);
4341 }
4342 EXPORT_SYMBOL_GPL(pci_check_and_mask_intx);
4343 
4344 /**
4345  * pci_check_and_unmask_intx - unmask INTx if no interrupt is pending
4346  * @dev: the PCI device to operate on
4347  *
4348  * Check if the device dev has its INTx line asserted, unmask it if not
4349  * and return true. False is returned and the mask remains active if
4350  * there was still an interrupt pending.
4351  */
pci_check_and_unmask_intx(struct pci_dev * dev)4352 bool pci_check_and_unmask_intx(struct pci_dev *dev)
4353 {
4354 	return pci_check_and_set_intx_mask(dev, false);
4355 }
4356 EXPORT_SYMBOL_GPL(pci_check_and_unmask_intx);
4357 
4358 /**
4359  * pci_wait_for_pending_transaction - waits for pending transaction
4360  * @dev: the PCI device to operate on
4361  *
4362  * Return 0 if transaction is pending 1 otherwise.
4363  */
pci_wait_for_pending_transaction(struct pci_dev * dev)4364 int pci_wait_for_pending_transaction(struct pci_dev *dev)
4365 {
4366 	if (!pci_is_pcie(dev))
4367 		return 1;
4368 
4369 	return pci_wait_for_pending(dev, pci_pcie_cap(dev) + PCI_EXP_DEVSTA,
4370 				    PCI_EXP_DEVSTA_TRPND);
4371 }
4372 EXPORT_SYMBOL(pci_wait_for_pending_transaction);
4373 
pci_dev_wait(struct pci_dev * dev,char * reset_type,int timeout)4374 static int pci_dev_wait(struct pci_dev *dev, char *reset_type, int timeout)
4375 {
4376 	int delay = 1;
4377 	u32 id;
4378 
4379 	/*
4380 	 * After reset, the device should not silently discard config
4381 	 * requests, but it may still indicate that it needs more time by
4382 	 * responding to them with CRS completions.  The Root Port will
4383 	 * generally synthesize ~0 data to complete the read (except when
4384 	 * CRS SV is enabled and the read was for the Vendor ID; in that
4385 	 * case it synthesizes 0x0001 data).
4386 	 *
4387 	 * Wait for the device to return a non-CRS completion.  Read the
4388 	 * Command register instead of Vendor ID so we don't have to
4389 	 * contend with the CRS SV value.
4390 	 */
4391 	pci_read_config_dword(dev, PCI_COMMAND, &id);
4392 	while (id == ~0) {
4393 		if (delay > timeout) {
4394 			pci_warn(dev, "not ready %dms after %s; giving up\n",
4395 				 delay - 1, reset_type);
4396 			return -ENOTTY;
4397 		}
4398 
4399 		if (delay > 1000)
4400 			pci_info(dev, "not ready %dms after %s; waiting\n",
4401 				 delay - 1, reset_type);
4402 
4403 		msleep(delay);
4404 		delay *= 2;
4405 		pci_read_config_dword(dev, PCI_COMMAND, &id);
4406 	}
4407 
4408 	if (delay > 1000)
4409 		pci_info(dev, "ready %dms after %s\n", delay - 1,
4410 			 reset_type);
4411 
4412 	return 0;
4413 }
4414 
4415 /**
4416  * pcie_has_flr - check if a device supports function level resets
4417  * @dev:	device to check
4418  *
4419  * Returns true if the device advertises support for PCIe function level
4420  * resets.
4421  */
pcie_has_flr(struct pci_dev * dev)4422 bool pcie_has_flr(struct pci_dev *dev)
4423 {
4424 	u32 cap;
4425 
4426 	if (dev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET)
4427 		return false;
4428 
4429 	pcie_capability_read_dword(dev, PCI_EXP_DEVCAP, &cap);
4430 	return cap & PCI_EXP_DEVCAP_FLR;
4431 }
4432 EXPORT_SYMBOL_GPL(pcie_has_flr);
4433 
4434 /**
4435  * pcie_flr - initiate a PCIe function level reset
4436  * @dev:	device to reset
4437  *
4438  * Initiate a function level reset on @dev.  The caller should ensure the
4439  * device supports FLR before calling this function, e.g. by using the
4440  * pcie_has_flr() helper.
4441  */
pcie_flr(struct pci_dev * dev)4442 int pcie_flr(struct pci_dev *dev)
4443 {
4444 	if (!pci_wait_for_pending_transaction(dev))
4445 		pci_err(dev, "timed out waiting for pending transaction; performing function level reset anyway\n");
4446 
4447 	pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR);
4448 
4449 	/*
4450 	 * Per PCIe r4.0, sec 6.6.2, a device must complete an FLR within
4451 	 * 100ms, but may silently discard requests while the FLR is in
4452 	 * progress.  Wait 100ms before trying to access the device.
4453 	 */
4454 	msleep(100);
4455 
4456 	return pci_dev_wait(dev, "FLR", PCIE_RESET_READY_POLL_MS);
4457 }
4458 EXPORT_SYMBOL_GPL(pcie_flr);
4459 
pci_af_flr(struct pci_dev * dev,int probe)4460 static int pci_af_flr(struct pci_dev *dev, int probe)
4461 {
4462 	int pos;
4463 	u8 cap;
4464 
4465 	pos = pci_find_capability(dev, PCI_CAP_ID_AF);
4466 	if (!pos)
4467 		return -ENOTTY;
4468 
4469 	if (dev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET)
4470 		return -ENOTTY;
4471 
4472 	pci_read_config_byte(dev, pos + PCI_AF_CAP, &cap);
4473 	if (!(cap & PCI_AF_CAP_TP) || !(cap & PCI_AF_CAP_FLR))
4474 		return -ENOTTY;
4475 
4476 	if (probe)
4477 		return 0;
4478 
4479 	/*
4480 	 * Wait for Transaction Pending bit to clear.  A word-aligned test
4481 	 * is used, so we use the conrol offset rather than status and shift
4482 	 * the test bit to match.
4483 	 */
4484 	if (!pci_wait_for_pending(dev, pos + PCI_AF_CTRL,
4485 				 PCI_AF_STATUS_TP << 8))
4486 		pci_err(dev, "timed out waiting for pending transaction; performing AF function level reset anyway\n");
4487 
4488 	pci_write_config_byte(dev, pos + PCI_AF_CTRL, PCI_AF_CTRL_FLR);
4489 
4490 	/*
4491 	 * Per Advanced Capabilities for Conventional PCI ECN, 13 April 2006,
4492 	 * updated 27 July 2006; a device must complete an FLR within
4493 	 * 100ms, but may silently discard requests while the FLR is in
4494 	 * progress.  Wait 100ms before trying to access the device.
4495 	 */
4496 	msleep(100);
4497 
4498 	return pci_dev_wait(dev, "AF_FLR", PCIE_RESET_READY_POLL_MS);
4499 }
4500 
4501 /**
4502  * pci_pm_reset - Put device into PCI_D3 and back into PCI_D0.
4503  * @dev: Device to reset.
4504  * @probe: If set, only check if the device can be reset this way.
4505  *
4506  * If @dev supports native PCI PM and its PCI_PM_CTRL_NO_SOFT_RESET flag is
4507  * unset, it will be reinitialized internally when going from PCI_D3hot to
4508  * PCI_D0.  If that's the case and the device is not in a low-power state
4509  * already, force it into PCI_D3hot and back to PCI_D0, causing it to be reset.
4510  *
4511  * NOTE: This causes the caller to sleep for twice the device power transition
4512  * cooldown period, which for the D0->D3hot and D3hot->D0 transitions is 10 ms
4513  * by default (i.e. unless the @dev's d3_delay field has a different value).
4514  * Moreover, only devices in D0 can be reset by this function.
4515  */
pci_pm_reset(struct pci_dev * dev,int probe)4516 static int pci_pm_reset(struct pci_dev *dev, int probe)
4517 {
4518 	u16 csr;
4519 
4520 	if (!dev->pm_cap || dev->dev_flags & PCI_DEV_FLAGS_NO_PM_RESET)
4521 		return -ENOTTY;
4522 
4523 	pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &csr);
4524 	if (csr & PCI_PM_CTRL_NO_SOFT_RESET)
4525 		return -ENOTTY;
4526 
4527 	if (probe)
4528 		return 0;
4529 
4530 	if (dev->current_state != PCI_D0)
4531 		return -EINVAL;
4532 
4533 	csr &= ~PCI_PM_CTRL_STATE_MASK;
4534 	csr |= PCI_D3hot;
4535 	pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, csr);
4536 	pci_dev_d3_sleep(dev);
4537 
4538 	csr &= ~PCI_PM_CTRL_STATE_MASK;
4539 	csr |= PCI_D0;
4540 	pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, csr);
4541 	pci_dev_d3_sleep(dev);
4542 
4543 	return pci_dev_wait(dev, "PM D3->D0", PCIE_RESET_READY_POLL_MS);
4544 }
4545 /**
4546  * pcie_wait_for_link - Wait until link is active or inactive
4547  * @pdev: Bridge device
4548  * @active: waiting for active or inactive?
4549  *
4550  * Use this to wait till link becomes active or inactive.
4551  */
pcie_wait_for_link(struct pci_dev * pdev,bool active)4552 bool pcie_wait_for_link(struct pci_dev *pdev, bool active)
4553 {
4554 	int timeout = 1000;
4555 	bool ret;
4556 	u16 lnk_status;
4557 
4558 	for (;;) {
4559 		pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnk_status);
4560 		ret = !!(lnk_status & PCI_EXP_LNKSTA_DLLLA);
4561 		if (ret == active)
4562 			return true;
4563 		if (timeout <= 0)
4564 			break;
4565 		msleep(10);
4566 		timeout -= 10;
4567 	}
4568 
4569 	pci_info(pdev, "Data Link Layer Link Active not %s in 1000 msec\n",
4570 		 active ? "set" : "cleared");
4571 
4572 	return false;
4573 }
4574 
pci_reset_secondary_bus(struct pci_dev * dev)4575 void pci_reset_secondary_bus(struct pci_dev *dev)
4576 {
4577 	u16 ctrl;
4578 
4579 	pci_read_config_word(dev, PCI_BRIDGE_CONTROL, &ctrl);
4580 	ctrl |= PCI_BRIDGE_CTL_BUS_RESET;
4581 	pci_write_config_word(dev, PCI_BRIDGE_CONTROL, ctrl);
4582 
4583 	/*
4584 	 * PCI spec v3.0 7.6.4.2 requires minimum Trst of 1ms.  Double
4585 	 * this to 2ms to ensure that we meet the minimum requirement.
4586 	 */
4587 	msleep(2);
4588 
4589 	ctrl &= ~PCI_BRIDGE_CTL_BUS_RESET;
4590 	pci_write_config_word(dev, PCI_BRIDGE_CONTROL, ctrl);
4591 
4592 	/*
4593 	 * Trhfa for conventional PCI is 2^25 clock cycles.
4594 	 * Assuming a minimum 33MHz clock this results in a 1s
4595 	 * delay before we can consider subordinate devices to
4596 	 * be re-initialized.  PCIe has some ways to shorten this,
4597 	 * but we don't make use of them yet.
4598 	 */
4599 	ssleep(1);
4600 }
4601 
pcibios_reset_secondary_bus(struct pci_dev * dev)4602 void __weak pcibios_reset_secondary_bus(struct pci_dev *dev)
4603 {
4604 	pci_reset_secondary_bus(dev);
4605 }
4606 
4607 /**
4608  * pci_bridge_secondary_bus_reset - Reset the secondary bus on a PCI bridge.
4609  * @dev: Bridge device
4610  *
4611  * Use the bridge control register to assert reset on the secondary bus.
4612  * Devices on the secondary bus are left in power-on state.
4613  */
pci_bridge_secondary_bus_reset(struct pci_dev * dev)4614 int pci_bridge_secondary_bus_reset(struct pci_dev *dev)
4615 {
4616 	pcibios_reset_secondary_bus(dev);
4617 
4618 	return pci_dev_wait(dev, "bus reset", PCIE_RESET_READY_POLL_MS);
4619 }
4620 EXPORT_SYMBOL_GPL(pci_bridge_secondary_bus_reset);
4621 
pci_parent_bus_reset(struct pci_dev * dev,int probe)4622 static int pci_parent_bus_reset(struct pci_dev *dev, int probe)
4623 {
4624 	struct pci_dev *pdev;
4625 
4626 	if (pci_is_root_bus(dev->bus) || dev->subordinate ||
4627 	    !dev->bus->self || dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET)
4628 		return -ENOTTY;
4629 
4630 	list_for_each_entry(pdev, &dev->bus->devices, bus_list)
4631 		if (pdev != dev)
4632 			return -ENOTTY;
4633 
4634 	if (probe)
4635 		return 0;
4636 
4637 	return pci_bridge_secondary_bus_reset(dev->bus->self);
4638 }
4639 
pci_reset_hotplug_slot(struct hotplug_slot * hotplug,int probe)4640 static int pci_reset_hotplug_slot(struct hotplug_slot *hotplug, int probe)
4641 {
4642 	int rc = -ENOTTY;
4643 
4644 	if (!hotplug || !try_module_get(hotplug->ops->owner))
4645 		return rc;
4646 
4647 	if (hotplug->ops->reset_slot)
4648 		rc = hotplug->ops->reset_slot(hotplug, probe);
4649 
4650 	module_put(hotplug->ops->owner);
4651 
4652 	return rc;
4653 }
4654 
pci_dev_reset_slot_function(struct pci_dev * dev,int probe)4655 static int pci_dev_reset_slot_function(struct pci_dev *dev, int probe)
4656 {
4657 	struct pci_dev *pdev;
4658 
4659 	if (dev->subordinate || !dev->slot ||
4660 	    dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET)
4661 		return -ENOTTY;
4662 
4663 	list_for_each_entry(pdev, &dev->bus->devices, bus_list)
4664 		if (pdev != dev && pdev->slot == dev->slot)
4665 			return -ENOTTY;
4666 
4667 	return pci_reset_hotplug_slot(dev->slot->hotplug, probe);
4668 }
4669 
pci_dev_lock(struct pci_dev * dev)4670 static void pci_dev_lock(struct pci_dev *dev)
4671 {
4672 	/* block PM suspend, driver probe, etc. */
4673 	device_lock(&dev->dev);
4674 	pci_cfg_access_lock(dev);
4675 }
4676 
4677 /* Return 1 on successful lock, 0 on contention */
pci_dev_trylock(struct pci_dev * dev)4678 static int pci_dev_trylock(struct pci_dev *dev)
4679 {
4680 	if (device_trylock(&dev->dev)) {
4681 		if (pci_cfg_access_trylock(dev))
4682 			return 1;
4683 		device_unlock(&dev->dev);
4684 	}
4685 
4686 	return 0;
4687 }
4688 
pci_dev_unlock(struct pci_dev * dev)4689 static void pci_dev_unlock(struct pci_dev *dev)
4690 {
4691 	pci_cfg_access_unlock(dev);
4692 	device_unlock(&dev->dev);
4693 }
4694 
pci_dev_save_and_disable(struct pci_dev * dev)4695 static void pci_dev_save_and_disable(struct pci_dev *dev)
4696 {
4697 	const struct pci_error_handlers *err_handler =
4698 			dev->driver ? dev->driver->err_handler : NULL;
4699 
4700 	/*
4701 	 * dev->driver->err_handler->reset_prepare() is protected against
4702 	 * races with ->remove() by the device lock, which must be held by
4703 	 * the caller.
4704 	 */
4705 	if (err_handler && err_handler->reset_prepare)
4706 		err_handler->reset_prepare(dev);
4707 
4708 	/*
4709 	 * Wake-up device prior to save.  PM registers default to D0 after
4710 	 * reset and a simple register restore doesn't reliably return
4711 	 * to a non-D0 state anyway.
4712 	 */
4713 	pci_set_power_state(dev, PCI_D0);
4714 
4715 	pci_save_state(dev);
4716 	/*
4717 	 * Disable the device by clearing the Command register, except for
4718 	 * INTx-disable which is set.  This not only disables MMIO and I/O port
4719 	 * BARs, but also prevents the device from being Bus Master, preventing
4720 	 * DMA from the device including MSI/MSI-X interrupts.  For PCI 2.3
4721 	 * compliant devices, INTx-disable prevents legacy interrupts.
4722 	 */
4723 	pci_write_config_word(dev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE);
4724 }
4725 
pci_dev_restore(struct pci_dev * dev)4726 static void pci_dev_restore(struct pci_dev *dev)
4727 {
4728 	const struct pci_error_handlers *err_handler =
4729 			dev->driver ? dev->driver->err_handler : NULL;
4730 
4731 	pci_restore_state(dev);
4732 
4733 	/*
4734 	 * dev->driver->err_handler->reset_done() is protected against
4735 	 * races with ->remove() by the device lock, which must be held by
4736 	 * the caller.
4737 	 */
4738 	if (err_handler && err_handler->reset_done)
4739 		err_handler->reset_done(dev);
4740 }
4741 
4742 /**
4743  * __pci_reset_function_locked - reset a PCI device function while holding
4744  * the @dev mutex lock.
4745  * @dev: PCI device to reset
4746  *
4747  * Some devices allow an individual function to be reset without affecting
4748  * other functions in the same device.  The PCI device must be responsive
4749  * to PCI config space in order to use this function.
4750  *
4751  * The device function is presumed to be unused and the caller is holding
4752  * the device mutex lock when this function is called.
4753  * Resetting the device will make the contents of PCI configuration space
4754  * random, so any caller of this must be prepared to reinitialise the
4755  * device including MSI, bus mastering, BARs, decoding IO and memory spaces,
4756  * etc.
4757  *
4758  * Returns 0 if the device function was successfully reset or negative if the
4759  * device doesn't support resetting a single function.
4760  */
__pci_reset_function_locked(struct pci_dev * dev)4761 int __pci_reset_function_locked(struct pci_dev *dev)
4762 {
4763 	int rc;
4764 
4765 	might_sleep();
4766 
4767 	/*
4768 	 * A reset method returns -ENOTTY if it doesn't support this device
4769 	 * and we should try the next method.
4770 	 *
4771 	 * If it returns 0 (success), we're finished.  If it returns any
4772 	 * other error, we're also finished: this indicates that further
4773 	 * reset mechanisms might be broken on the device.
4774 	 */
4775 	rc = pci_dev_specific_reset(dev, 0);
4776 	if (rc != -ENOTTY)
4777 		return rc;
4778 	if (pcie_has_flr(dev)) {
4779 		rc = pcie_flr(dev);
4780 		if (rc != -ENOTTY)
4781 			return rc;
4782 	}
4783 	rc = pci_af_flr(dev, 0);
4784 	if (rc != -ENOTTY)
4785 		return rc;
4786 	rc = pci_pm_reset(dev, 0);
4787 	if (rc != -ENOTTY)
4788 		return rc;
4789 	rc = pci_dev_reset_slot_function(dev, 0);
4790 	if (rc != -ENOTTY)
4791 		return rc;
4792 	return pci_parent_bus_reset(dev, 0);
4793 }
4794 EXPORT_SYMBOL_GPL(__pci_reset_function_locked);
4795 
4796 /**
4797  * pci_probe_reset_function - check whether the device can be safely reset
4798  * @dev: PCI device to reset
4799  *
4800  * Some devices allow an individual function to be reset without affecting
4801  * other functions in the same device.  The PCI device must be responsive
4802  * to PCI config space in order to use this function.
4803  *
4804  * Returns 0 if the device function can be reset or negative if the
4805  * device doesn't support resetting a single function.
4806  */
pci_probe_reset_function(struct pci_dev * dev)4807 int pci_probe_reset_function(struct pci_dev *dev)
4808 {
4809 	int rc;
4810 
4811 	might_sleep();
4812 
4813 	rc = pci_dev_specific_reset(dev, 1);
4814 	if (rc != -ENOTTY)
4815 		return rc;
4816 	if (pcie_has_flr(dev))
4817 		return 0;
4818 	rc = pci_af_flr(dev, 1);
4819 	if (rc != -ENOTTY)
4820 		return rc;
4821 	rc = pci_pm_reset(dev, 1);
4822 	if (rc != -ENOTTY)
4823 		return rc;
4824 	rc = pci_dev_reset_slot_function(dev, 1);
4825 	if (rc != -ENOTTY)
4826 		return rc;
4827 
4828 	return pci_parent_bus_reset(dev, 1);
4829 }
4830 
4831 /**
4832  * pci_reset_function - quiesce and reset a PCI device function
4833  * @dev: PCI device to reset
4834  *
4835  * Some devices allow an individual function to be reset without affecting
4836  * other functions in the same device.  The PCI device must be responsive
4837  * to PCI config space in order to use this function.
4838  *
4839  * This function does not just reset the PCI portion of a device, but
4840  * clears all the state associated with the device.  This function differs
4841  * from __pci_reset_function_locked() in that it saves and restores device state
4842  * over the reset and takes the PCI device lock.
4843  *
4844  * Returns 0 if the device function was successfully reset or negative if the
4845  * device doesn't support resetting a single function.
4846  */
pci_reset_function(struct pci_dev * dev)4847 int pci_reset_function(struct pci_dev *dev)
4848 {
4849 	int rc;
4850 
4851 	if (!dev->reset_fn)
4852 		return -ENOTTY;
4853 
4854 	pci_dev_lock(dev);
4855 	pci_dev_save_and_disable(dev);
4856 
4857 	rc = __pci_reset_function_locked(dev);
4858 
4859 	pci_dev_restore(dev);
4860 	pci_dev_unlock(dev);
4861 
4862 	return rc;
4863 }
4864 EXPORT_SYMBOL_GPL(pci_reset_function);
4865 
4866 /**
4867  * pci_reset_function_locked - quiesce and reset a PCI device function
4868  * @dev: PCI device to reset
4869  *
4870  * Some devices allow an individual function to be reset without affecting
4871  * other functions in the same device.  The PCI device must be responsive
4872  * to PCI config space in order to use this function.
4873  *
4874  * This function does not just reset the PCI portion of a device, but
4875  * clears all the state associated with the device.  This function differs
4876  * from __pci_reset_function_locked() in that it saves and restores device state
4877  * over the reset.  It also differs from pci_reset_function() in that it
4878  * requires the PCI device lock to be held.
4879  *
4880  * Returns 0 if the device function was successfully reset or negative if the
4881  * device doesn't support resetting a single function.
4882  */
pci_reset_function_locked(struct pci_dev * dev)4883 int pci_reset_function_locked(struct pci_dev *dev)
4884 {
4885 	int rc;
4886 
4887 	if (!dev->reset_fn)
4888 		return -ENOTTY;
4889 
4890 	pci_dev_save_and_disable(dev);
4891 
4892 	rc = __pci_reset_function_locked(dev);
4893 
4894 	pci_dev_restore(dev);
4895 
4896 	return rc;
4897 }
4898 EXPORT_SYMBOL_GPL(pci_reset_function_locked);
4899 
4900 /**
4901  * pci_try_reset_function - quiesce and reset a PCI device function
4902  * @dev: PCI device to reset
4903  *
4904  * Same as above, except return -EAGAIN if unable to lock device.
4905  */
pci_try_reset_function(struct pci_dev * dev)4906 int pci_try_reset_function(struct pci_dev *dev)
4907 {
4908 	int rc;
4909 
4910 	if (!dev->reset_fn)
4911 		return -ENOTTY;
4912 
4913 	if (!pci_dev_trylock(dev))
4914 		return -EAGAIN;
4915 
4916 	pci_dev_save_and_disable(dev);
4917 	rc = __pci_reset_function_locked(dev);
4918 	pci_dev_restore(dev);
4919 	pci_dev_unlock(dev);
4920 
4921 	return rc;
4922 }
4923 EXPORT_SYMBOL_GPL(pci_try_reset_function);
4924 
4925 /* Do any devices on or below this bus prevent a bus reset? */
pci_bus_resetable(struct pci_bus * bus)4926 static bool pci_bus_resetable(struct pci_bus *bus)
4927 {
4928 	struct pci_dev *dev;
4929 
4930 
4931 	if (bus->self && (bus->self->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET))
4932 		return false;
4933 
4934 	list_for_each_entry(dev, &bus->devices, bus_list) {
4935 		if (dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET ||
4936 		    (dev->subordinate && !pci_bus_resetable(dev->subordinate)))
4937 			return false;
4938 	}
4939 
4940 	return true;
4941 }
4942 
4943 /* Lock devices from the top of the tree down */
pci_bus_lock(struct pci_bus * bus)4944 static void pci_bus_lock(struct pci_bus *bus)
4945 {
4946 	struct pci_dev *dev;
4947 
4948 	list_for_each_entry(dev, &bus->devices, bus_list) {
4949 		pci_dev_lock(dev);
4950 		if (dev->subordinate)
4951 			pci_bus_lock(dev->subordinate);
4952 	}
4953 }
4954 
4955 /* Unlock devices from the bottom of the tree up */
pci_bus_unlock(struct pci_bus * bus)4956 static void pci_bus_unlock(struct pci_bus *bus)
4957 {
4958 	struct pci_dev *dev;
4959 
4960 	list_for_each_entry(dev, &bus->devices, bus_list) {
4961 		if (dev->subordinate)
4962 			pci_bus_unlock(dev->subordinate);
4963 		pci_dev_unlock(dev);
4964 	}
4965 }
4966 
4967 /* Return 1 on successful lock, 0 on contention */
pci_bus_trylock(struct pci_bus * bus)4968 static int pci_bus_trylock(struct pci_bus *bus)
4969 {
4970 	struct pci_dev *dev;
4971 
4972 	list_for_each_entry(dev, &bus->devices, bus_list) {
4973 		if (!pci_dev_trylock(dev))
4974 			goto unlock;
4975 		if (dev->subordinate) {
4976 			if (!pci_bus_trylock(dev->subordinate)) {
4977 				pci_dev_unlock(dev);
4978 				goto unlock;
4979 			}
4980 		}
4981 	}
4982 	return 1;
4983 
4984 unlock:
4985 	list_for_each_entry_continue_reverse(dev, &bus->devices, bus_list) {
4986 		if (dev->subordinate)
4987 			pci_bus_unlock(dev->subordinate);
4988 		pci_dev_unlock(dev);
4989 	}
4990 	return 0;
4991 }
4992 
4993 /* Do any devices on or below this slot prevent a bus reset? */
pci_slot_resetable(struct pci_slot * slot)4994 static bool pci_slot_resetable(struct pci_slot *slot)
4995 {
4996 	struct pci_dev *dev;
4997 
4998 	if (slot->bus->self &&
4999 	    (slot->bus->self->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET))
5000 		return false;
5001 
5002 	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
5003 		if (!dev->slot || dev->slot != slot)
5004 			continue;
5005 		if (dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET ||
5006 		    (dev->subordinate && !pci_bus_resetable(dev->subordinate)))
5007 			return false;
5008 	}
5009 
5010 	return true;
5011 }
5012 
5013 /* Lock devices from the top of the tree down */
pci_slot_lock(struct pci_slot * slot)5014 static void pci_slot_lock(struct pci_slot *slot)
5015 {
5016 	struct pci_dev *dev;
5017 
5018 	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
5019 		if (!dev->slot || dev->slot != slot)
5020 			continue;
5021 		pci_dev_lock(dev);
5022 		if (dev->subordinate)
5023 			pci_bus_lock(dev->subordinate);
5024 	}
5025 }
5026 
5027 /* Unlock devices from the bottom of the tree up */
pci_slot_unlock(struct pci_slot * slot)5028 static void pci_slot_unlock(struct pci_slot *slot)
5029 {
5030 	struct pci_dev *dev;
5031 
5032 	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
5033 		if (!dev->slot || dev->slot != slot)
5034 			continue;
5035 		if (dev->subordinate)
5036 			pci_bus_unlock(dev->subordinate);
5037 		pci_dev_unlock(dev);
5038 	}
5039 }
5040 
5041 /* Return 1 on successful lock, 0 on contention */
pci_slot_trylock(struct pci_slot * slot)5042 static int pci_slot_trylock(struct pci_slot *slot)
5043 {
5044 	struct pci_dev *dev;
5045 
5046 	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
5047 		if (!dev->slot || dev->slot != slot)
5048 			continue;
5049 		if (!pci_dev_trylock(dev))
5050 			goto unlock;
5051 		if (dev->subordinate) {
5052 			if (!pci_bus_trylock(dev->subordinate)) {
5053 				pci_dev_unlock(dev);
5054 				goto unlock;
5055 			}
5056 		}
5057 	}
5058 	return 1;
5059 
5060 unlock:
5061 	list_for_each_entry_continue_reverse(dev,
5062 					     &slot->bus->devices, bus_list) {
5063 		if (!dev->slot || dev->slot != slot)
5064 			continue;
5065 		if (dev->subordinate)
5066 			pci_bus_unlock(dev->subordinate);
5067 		pci_dev_unlock(dev);
5068 	}
5069 	return 0;
5070 }
5071 
5072 /*
5073  * Save and disable devices from the top of the tree down while holding
5074  * the @dev mutex lock for the entire tree.
5075  */
pci_bus_save_and_disable_locked(struct pci_bus * bus)5076 static void pci_bus_save_and_disable_locked(struct pci_bus *bus)
5077 {
5078 	struct pci_dev *dev;
5079 
5080 	list_for_each_entry(dev, &bus->devices, bus_list) {
5081 		pci_dev_save_and_disable(dev);
5082 		if (dev->subordinate)
5083 			pci_bus_save_and_disable_locked(dev->subordinate);
5084 	}
5085 }
5086 
5087 /*
5088  * Restore devices from top of the tree down while holding @dev mutex lock
5089  * for the entire tree.  Parent bridges need to be restored before we can
5090  * get to subordinate devices.
5091  */
pci_bus_restore_locked(struct pci_bus * bus)5092 static void pci_bus_restore_locked(struct pci_bus *bus)
5093 {
5094 	struct pci_dev *dev;
5095 
5096 	list_for_each_entry(dev, &bus->devices, bus_list) {
5097 		pci_dev_restore(dev);
5098 		if (dev->subordinate)
5099 			pci_bus_restore_locked(dev->subordinate);
5100 	}
5101 }
5102 
5103 /*
5104  * Save and disable devices from the top of the tree down while holding
5105  * the @dev mutex lock for the entire tree.
5106  */
pci_slot_save_and_disable_locked(struct pci_slot * slot)5107 static void pci_slot_save_and_disable_locked(struct pci_slot *slot)
5108 {
5109 	struct pci_dev *dev;
5110 
5111 	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
5112 		if (!dev->slot || dev->slot != slot)
5113 			continue;
5114 		pci_dev_save_and_disable(dev);
5115 		if (dev->subordinate)
5116 			pci_bus_save_and_disable_locked(dev->subordinate);
5117 	}
5118 }
5119 
5120 /*
5121  * Restore devices from top of the tree down while holding @dev mutex lock
5122  * for the entire tree.  Parent bridges need to be restored before we can
5123  * get to subordinate devices.
5124  */
pci_slot_restore_locked(struct pci_slot * slot)5125 static void pci_slot_restore_locked(struct pci_slot *slot)
5126 {
5127 	struct pci_dev *dev;
5128 
5129 	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
5130 		if (!dev->slot || dev->slot != slot)
5131 			continue;
5132 		pci_dev_restore(dev);
5133 		if (dev->subordinate)
5134 			pci_bus_restore_locked(dev->subordinate);
5135 	}
5136 }
5137 
pci_slot_reset(struct pci_slot * slot,int probe)5138 static int pci_slot_reset(struct pci_slot *slot, int probe)
5139 {
5140 	int rc;
5141 
5142 	if (!slot || !pci_slot_resetable(slot))
5143 		return -ENOTTY;
5144 
5145 	if (!probe)
5146 		pci_slot_lock(slot);
5147 
5148 	might_sleep();
5149 
5150 	rc = pci_reset_hotplug_slot(slot->hotplug, probe);
5151 
5152 	if (!probe)
5153 		pci_slot_unlock(slot);
5154 
5155 	return rc;
5156 }
5157 
5158 /**
5159  * pci_probe_reset_slot - probe whether a PCI slot can be reset
5160  * @slot: PCI slot to probe
5161  *
5162  * Return 0 if slot can be reset, negative if a slot reset is not supported.
5163  */
pci_probe_reset_slot(struct pci_slot * slot)5164 int pci_probe_reset_slot(struct pci_slot *slot)
5165 {
5166 	return pci_slot_reset(slot, 1);
5167 }
5168 EXPORT_SYMBOL_GPL(pci_probe_reset_slot);
5169 
5170 /**
5171  * __pci_reset_slot - Try to reset a PCI slot
5172  * @slot: PCI slot to reset
5173  *
5174  * A PCI bus may host multiple slots, each slot may support a reset mechanism
5175  * independent of other slots.  For instance, some slots may support slot power
5176  * control.  In the case of a 1:1 bus to slot architecture, this function may
5177  * wrap the bus reset to avoid spurious slot related events such as hotplug.
5178  * Generally a slot reset should be attempted before a bus reset.  All of the
5179  * function of the slot and any subordinate buses behind the slot are reset
5180  * through this function.  PCI config space of all devices in the slot and
5181  * behind the slot is saved before and restored after reset.
5182  *
5183  * Same as above except return -EAGAIN if the slot cannot be locked
5184  */
__pci_reset_slot(struct pci_slot * slot)5185 static int __pci_reset_slot(struct pci_slot *slot)
5186 {
5187 	int rc;
5188 
5189 	rc = pci_slot_reset(slot, 1);
5190 	if (rc)
5191 		return rc;
5192 
5193 	if (pci_slot_trylock(slot)) {
5194 		pci_slot_save_and_disable_locked(slot);
5195 		might_sleep();
5196 		rc = pci_reset_hotplug_slot(slot->hotplug, 0);
5197 		pci_slot_restore_locked(slot);
5198 		pci_slot_unlock(slot);
5199 	} else
5200 		rc = -EAGAIN;
5201 
5202 	return rc;
5203 }
5204 
pci_bus_reset(struct pci_bus * bus,int probe)5205 static int pci_bus_reset(struct pci_bus *bus, int probe)
5206 {
5207 	int ret;
5208 
5209 	if (!bus->self || !pci_bus_resetable(bus))
5210 		return -ENOTTY;
5211 
5212 	if (probe)
5213 		return 0;
5214 
5215 	pci_bus_lock(bus);
5216 
5217 	might_sleep();
5218 
5219 	ret = pci_bridge_secondary_bus_reset(bus->self);
5220 
5221 	pci_bus_unlock(bus);
5222 
5223 	return ret;
5224 }
5225 
5226 /**
5227  * pci_bus_error_reset - reset the bridge's subordinate bus
5228  * @bridge: The parent device that connects to the bus to reset
5229  *
5230  * This function will first try to reset the slots on this bus if the method is
5231  * available. If slot reset fails or is not available, this will fall back to a
5232  * secondary bus reset.
5233  */
pci_bus_error_reset(struct pci_dev * bridge)5234 int pci_bus_error_reset(struct pci_dev *bridge)
5235 {
5236 	struct pci_bus *bus = bridge->subordinate;
5237 	struct pci_slot *slot;
5238 
5239 	if (!bus)
5240 		return -ENOTTY;
5241 
5242 	mutex_lock(&pci_slot_mutex);
5243 	if (list_empty(&bus->slots))
5244 		goto bus_reset;
5245 
5246 	list_for_each_entry(slot, &bus->slots, list)
5247 		if (pci_probe_reset_slot(slot))
5248 			goto bus_reset;
5249 
5250 	list_for_each_entry(slot, &bus->slots, list)
5251 		if (pci_slot_reset(slot, 0))
5252 			goto bus_reset;
5253 
5254 	mutex_unlock(&pci_slot_mutex);
5255 	return 0;
5256 bus_reset:
5257 	mutex_unlock(&pci_slot_mutex);
5258 	return pci_bus_reset(bridge->subordinate, 0);
5259 }
5260 
5261 /**
5262  * pci_probe_reset_bus - probe whether a PCI bus can be reset
5263  * @bus: PCI bus to probe
5264  *
5265  * Return 0 if bus can be reset, negative if a bus reset is not supported.
5266  */
pci_probe_reset_bus(struct pci_bus * bus)5267 int pci_probe_reset_bus(struct pci_bus *bus)
5268 {
5269 	return pci_bus_reset(bus, 1);
5270 }
5271 EXPORT_SYMBOL_GPL(pci_probe_reset_bus);
5272 
5273 /**
5274  * __pci_reset_bus - Try to reset a PCI bus
5275  * @bus: top level PCI bus to reset
5276  *
5277  * Same as above except return -EAGAIN if the bus cannot be locked
5278  */
__pci_reset_bus(struct pci_bus * bus)5279 static int __pci_reset_bus(struct pci_bus *bus)
5280 {
5281 	int rc;
5282 
5283 	rc = pci_bus_reset(bus, 1);
5284 	if (rc)
5285 		return rc;
5286 
5287 	if (pci_bus_trylock(bus)) {
5288 		pci_bus_save_and_disable_locked(bus);
5289 		might_sleep();
5290 		rc = pci_bridge_secondary_bus_reset(bus->self);
5291 		pci_bus_restore_locked(bus);
5292 		pci_bus_unlock(bus);
5293 	} else
5294 		rc = -EAGAIN;
5295 
5296 	return rc;
5297 }
5298 
5299 /**
5300  * pci_reset_bus - Try to reset a PCI bus
5301  * @pdev: top level PCI device to reset via slot/bus
5302  *
5303  * Same as above except return -EAGAIN if the bus cannot be locked
5304  */
pci_reset_bus(struct pci_dev * pdev)5305 int pci_reset_bus(struct pci_dev *pdev)
5306 {
5307 	return (!pci_probe_reset_slot(pdev->slot)) ?
5308 	    __pci_reset_slot(pdev->slot) : __pci_reset_bus(pdev->bus);
5309 }
5310 EXPORT_SYMBOL_GPL(pci_reset_bus);
5311 
5312 /**
5313  * pcix_get_max_mmrbc - get PCI-X maximum designed memory read byte count
5314  * @dev: PCI device to query
5315  *
5316  * Returns mmrbc: maximum designed memory read count in bytes
5317  *    or appropriate error value.
5318  */
pcix_get_max_mmrbc(struct pci_dev * dev)5319 int pcix_get_max_mmrbc(struct pci_dev *dev)
5320 {
5321 	int cap;
5322 	u32 stat;
5323 
5324 	cap = pci_find_capability(dev, PCI_CAP_ID_PCIX);
5325 	if (!cap)
5326 		return -EINVAL;
5327 
5328 	if (pci_read_config_dword(dev, cap + PCI_X_STATUS, &stat))
5329 		return -EINVAL;
5330 
5331 	return 512 << ((stat & PCI_X_STATUS_MAX_READ) >> 21);
5332 }
5333 EXPORT_SYMBOL(pcix_get_max_mmrbc);
5334 
5335 /**
5336  * pcix_get_mmrbc - get PCI-X maximum memory read byte count
5337  * @dev: PCI device to query
5338  *
5339  * Returns mmrbc: maximum memory read count in bytes
5340  *    or appropriate error value.
5341  */
pcix_get_mmrbc(struct pci_dev * dev)5342 int pcix_get_mmrbc(struct pci_dev *dev)
5343 {
5344 	int cap;
5345 	u16 cmd;
5346 
5347 	cap = pci_find_capability(dev, PCI_CAP_ID_PCIX);
5348 	if (!cap)
5349 		return -EINVAL;
5350 
5351 	if (pci_read_config_word(dev, cap + PCI_X_CMD, &cmd))
5352 		return -EINVAL;
5353 
5354 	return 512 << ((cmd & PCI_X_CMD_MAX_READ) >> 2);
5355 }
5356 EXPORT_SYMBOL(pcix_get_mmrbc);
5357 
5358 /**
5359  * pcix_set_mmrbc - set PCI-X maximum memory read byte count
5360  * @dev: PCI device to query
5361  * @mmrbc: maximum memory read count in bytes
5362  *    valid values are 512, 1024, 2048, 4096
5363  *
5364  * If possible sets maximum memory read byte count, some bridges have erratas
5365  * that prevent this.
5366  */
pcix_set_mmrbc(struct pci_dev * dev,int mmrbc)5367 int pcix_set_mmrbc(struct pci_dev *dev, int mmrbc)
5368 {
5369 	int cap;
5370 	u32 stat, v, o;
5371 	u16 cmd;
5372 
5373 	if (mmrbc < 512 || mmrbc > 4096 || !is_power_of_2(mmrbc))
5374 		return -EINVAL;
5375 
5376 	v = ffs(mmrbc) - 10;
5377 
5378 	cap = pci_find_capability(dev, PCI_CAP_ID_PCIX);
5379 	if (!cap)
5380 		return -EINVAL;
5381 
5382 	if (pci_read_config_dword(dev, cap + PCI_X_STATUS, &stat))
5383 		return -EINVAL;
5384 
5385 	if (v > (stat & PCI_X_STATUS_MAX_READ) >> 21)
5386 		return -E2BIG;
5387 
5388 	if (pci_read_config_word(dev, cap + PCI_X_CMD, &cmd))
5389 		return -EINVAL;
5390 
5391 	o = (cmd & PCI_X_CMD_MAX_READ) >> 2;
5392 	if (o != v) {
5393 		if (v > o && (dev->bus->bus_flags & PCI_BUS_FLAGS_NO_MMRBC))
5394 			return -EIO;
5395 
5396 		cmd &= ~PCI_X_CMD_MAX_READ;
5397 		cmd |= v << 2;
5398 		if (pci_write_config_word(dev, cap + PCI_X_CMD, cmd))
5399 			return -EIO;
5400 	}
5401 	return 0;
5402 }
5403 EXPORT_SYMBOL(pcix_set_mmrbc);
5404 
5405 /**
5406  * pcie_get_readrq - get PCI Express read request size
5407  * @dev: PCI device to query
5408  *
5409  * Returns maximum memory read request in bytes
5410  *    or appropriate error value.
5411  */
pcie_get_readrq(struct pci_dev * dev)5412 int pcie_get_readrq(struct pci_dev *dev)
5413 {
5414 	u16 ctl;
5415 
5416 	pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &ctl);
5417 
5418 	return 128 << ((ctl & PCI_EXP_DEVCTL_READRQ) >> 12);
5419 }
5420 EXPORT_SYMBOL(pcie_get_readrq);
5421 
5422 /**
5423  * pcie_set_readrq - set PCI Express maximum memory read request
5424  * @dev: PCI device to query
5425  * @rq: maximum memory read count in bytes
5426  *    valid values are 128, 256, 512, 1024, 2048, 4096
5427  *
5428  * If possible sets maximum memory read request in bytes
5429  */
pcie_set_readrq(struct pci_dev * dev,int rq)5430 int pcie_set_readrq(struct pci_dev *dev, int rq)
5431 {
5432 	u16 v;
5433 
5434 	if (rq < 128 || rq > 4096 || !is_power_of_2(rq))
5435 		return -EINVAL;
5436 
5437 	/*
5438 	 * If using the "performance" PCIe config, we clamp the
5439 	 * read rq size to the max packet size to prevent the
5440 	 * host bridge generating requests larger than we can
5441 	 * cope with
5442 	 */
5443 	if (pcie_bus_config == PCIE_BUS_PERFORMANCE) {
5444 		int mps = pcie_get_mps(dev);
5445 
5446 		if (mps < rq)
5447 			rq = mps;
5448 	}
5449 
5450 	v = (ffs(rq) - 8) << 12;
5451 
5452 	return pcie_capability_clear_and_set_word(dev, PCI_EXP_DEVCTL,
5453 						  PCI_EXP_DEVCTL_READRQ, v);
5454 }
5455 EXPORT_SYMBOL(pcie_set_readrq);
5456 
5457 /**
5458  * pcie_get_mps - get PCI Express maximum payload size
5459  * @dev: PCI device to query
5460  *
5461  * Returns maximum payload size in bytes
5462  */
pcie_get_mps(struct pci_dev * dev)5463 int pcie_get_mps(struct pci_dev *dev)
5464 {
5465 	u16 ctl;
5466 
5467 	pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &ctl);
5468 
5469 	return 128 << ((ctl & PCI_EXP_DEVCTL_PAYLOAD) >> 5);
5470 }
5471 EXPORT_SYMBOL(pcie_get_mps);
5472 
5473 /**
5474  * pcie_set_mps - set PCI Express maximum payload size
5475  * @dev: PCI device to query
5476  * @mps: maximum payload size in bytes
5477  *    valid values are 128, 256, 512, 1024, 2048, 4096
5478  *
5479  * If possible sets maximum payload size
5480  */
pcie_set_mps(struct pci_dev * dev,int mps)5481 int pcie_set_mps(struct pci_dev *dev, int mps)
5482 {
5483 	u16 v;
5484 
5485 	if (mps < 128 || mps > 4096 || !is_power_of_2(mps))
5486 		return -EINVAL;
5487 
5488 	v = ffs(mps) - 8;
5489 	if (v > dev->pcie_mpss)
5490 		return -EINVAL;
5491 	v <<= 5;
5492 
5493 	return pcie_capability_clear_and_set_word(dev, PCI_EXP_DEVCTL,
5494 						  PCI_EXP_DEVCTL_PAYLOAD, v);
5495 }
5496 EXPORT_SYMBOL(pcie_set_mps);
5497 
5498 /**
5499  * pcie_bandwidth_available - determine minimum link settings of a PCIe
5500  *			      device and its bandwidth limitation
5501  * @dev: PCI device to query
5502  * @limiting_dev: storage for device causing the bandwidth limitation
5503  * @speed: storage for speed of limiting device
5504  * @width: storage for width of limiting device
5505  *
5506  * Walk up the PCI device chain and find the point where the minimum
5507  * bandwidth is available.  Return the bandwidth available there and (if
5508  * limiting_dev, speed, and width pointers are supplied) information about
5509  * that point.  The bandwidth returned is in Mb/s, i.e., megabits/second of
5510  * raw bandwidth.
5511  */
pcie_bandwidth_available(struct pci_dev * dev,struct pci_dev ** limiting_dev,enum pci_bus_speed * speed,enum pcie_link_width * width)5512 u32 pcie_bandwidth_available(struct pci_dev *dev, struct pci_dev **limiting_dev,
5513 			     enum pci_bus_speed *speed,
5514 			     enum pcie_link_width *width)
5515 {
5516 	u16 lnksta;
5517 	enum pci_bus_speed next_speed;
5518 	enum pcie_link_width next_width;
5519 	u32 bw, next_bw;
5520 
5521 	if (speed)
5522 		*speed = PCI_SPEED_UNKNOWN;
5523 	if (width)
5524 		*width = PCIE_LNK_WIDTH_UNKNOWN;
5525 
5526 	bw = 0;
5527 
5528 	while (dev) {
5529 		pcie_capability_read_word(dev, PCI_EXP_LNKSTA, &lnksta);
5530 
5531 		next_speed = pcie_link_speed[lnksta & PCI_EXP_LNKSTA_CLS];
5532 		next_width = (lnksta & PCI_EXP_LNKSTA_NLW) >>
5533 			PCI_EXP_LNKSTA_NLW_SHIFT;
5534 
5535 		next_bw = next_width * PCIE_SPEED2MBS_ENC(next_speed);
5536 
5537 		/* Check if current device limits the total bandwidth */
5538 		if (!bw || next_bw <= bw) {
5539 			bw = next_bw;
5540 
5541 			if (limiting_dev)
5542 				*limiting_dev = dev;
5543 			if (speed)
5544 				*speed = next_speed;
5545 			if (width)
5546 				*width = next_width;
5547 		}
5548 
5549 		dev = pci_upstream_bridge(dev);
5550 	}
5551 
5552 	return bw;
5553 }
5554 EXPORT_SYMBOL(pcie_bandwidth_available);
5555 
5556 /**
5557  * pcie_get_speed_cap - query for the PCI device's link speed capability
5558  * @dev: PCI device to query
5559  *
5560  * Query the PCI device speed capability.  Return the maximum link speed
5561  * supported by the device.
5562  */
pcie_get_speed_cap(struct pci_dev * dev)5563 enum pci_bus_speed pcie_get_speed_cap(struct pci_dev *dev)
5564 {
5565 	u32 lnkcap2, lnkcap;
5566 
5567 	/*
5568 	 * Link Capabilities 2 was added in PCIe r3.0, sec 7.8.18.  The
5569 	 * implementation note there recommends using the Supported Link
5570 	 * Speeds Vector in Link Capabilities 2 when supported.
5571 	 *
5572 	 * Without Link Capabilities 2, i.e., prior to PCIe r3.0, software
5573 	 * should use the Supported Link Speeds field in Link Capabilities,
5574 	 * where only 2.5 GT/s and 5.0 GT/s speeds were defined.
5575 	 */
5576 	pcie_capability_read_dword(dev, PCI_EXP_LNKCAP2, &lnkcap2);
5577 	if (lnkcap2) { /* PCIe r3.0-compliant */
5578 		if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_32_0GB)
5579 			return PCIE_SPEED_32_0GT;
5580 		else if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_16_0GB)
5581 			return PCIE_SPEED_16_0GT;
5582 		else if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_8_0GB)
5583 			return PCIE_SPEED_8_0GT;
5584 		else if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_5_0GB)
5585 			return PCIE_SPEED_5_0GT;
5586 		else if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_2_5GB)
5587 			return PCIE_SPEED_2_5GT;
5588 		return PCI_SPEED_UNKNOWN;
5589 	}
5590 
5591 	pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, &lnkcap);
5592 	if ((lnkcap & PCI_EXP_LNKCAP_SLS) == PCI_EXP_LNKCAP_SLS_5_0GB)
5593 		return PCIE_SPEED_5_0GT;
5594 	else if ((lnkcap & PCI_EXP_LNKCAP_SLS) == PCI_EXP_LNKCAP_SLS_2_5GB)
5595 		return PCIE_SPEED_2_5GT;
5596 
5597 	return PCI_SPEED_UNKNOWN;
5598 }
5599 EXPORT_SYMBOL(pcie_get_speed_cap);
5600 
5601 /**
5602  * pcie_get_width_cap - query for the PCI device's link width capability
5603  * @dev: PCI device to query
5604  *
5605  * Query the PCI device width capability.  Return the maximum link width
5606  * supported by the device.
5607  */
pcie_get_width_cap(struct pci_dev * dev)5608 enum pcie_link_width pcie_get_width_cap(struct pci_dev *dev)
5609 {
5610 	u32 lnkcap;
5611 
5612 	pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, &lnkcap);
5613 	if (lnkcap)
5614 		return (lnkcap & PCI_EXP_LNKCAP_MLW) >> 4;
5615 
5616 	return PCIE_LNK_WIDTH_UNKNOWN;
5617 }
5618 EXPORT_SYMBOL(pcie_get_width_cap);
5619 
5620 /**
5621  * pcie_bandwidth_capable - calculate a PCI device's link bandwidth capability
5622  * @dev: PCI device
5623  * @speed: storage for link speed
5624  * @width: storage for link width
5625  *
5626  * Calculate a PCI device's link bandwidth by querying for its link speed
5627  * and width, multiplying them, and applying encoding overhead.  The result
5628  * is in Mb/s, i.e., megabits/second of raw bandwidth.
5629  */
pcie_bandwidth_capable(struct pci_dev * dev,enum pci_bus_speed * speed,enum pcie_link_width * width)5630 u32 pcie_bandwidth_capable(struct pci_dev *dev, enum pci_bus_speed *speed,
5631 			   enum pcie_link_width *width)
5632 {
5633 	*speed = pcie_get_speed_cap(dev);
5634 	*width = pcie_get_width_cap(dev);
5635 
5636 	if (*speed == PCI_SPEED_UNKNOWN || *width == PCIE_LNK_WIDTH_UNKNOWN)
5637 		return 0;
5638 
5639 	return *width * PCIE_SPEED2MBS_ENC(*speed);
5640 }
5641 
5642 /**
5643  * __pcie_print_link_status - Report the PCI device's link speed and width
5644  * @dev: PCI device to query
5645  * @verbose: Print info even when enough bandwidth is available
5646  *
5647  * If the available bandwidth at the device is less than the device is
5648  * capable of, report the device's maximum possible bandwidth and the
5649  * upstream link that limits its performance.  If @verbose, always print
5650  * the available bandwidth, even if the device isn't constrained.
5651  */
__pcie_print_link_status(struct pci_dev * dev,bool verbose)5652 void __pcie_print_link_status(struct pci_dev *dev, bool verbose)
5653 {
5654 	enum pcie_link_width width, width_cap;
5655 	enum pci_bus_speed speed, speed_cap;
5656 	struct pci_dev *limiting_dev = NULL;
5657 	u32 bw_avail, bw_cap;
5658 
5659 	bw_cap = pcie_bandwidth_capable(dev, &speed_cap, &width_cap);
5660 	bw_avail = pcie_bandwidth_available(dev, &limiting_dev, &speed, &width);
5661 
5662 	if (bw_avail >= bw_cap && verbose)
5663 		pci_info(dev, "%u.%03u Gb/s available PCIe bandwidth (%s x%d link)\n",
5664 			 bw_cap / 1000, bw_cap % 1000,
5665 			 PCIE_SPEED2STR(speed_cap), width_cap);
5666 	else if (bw_avail < bw_cap)
5667 		pci_info(dev, "%u.%03u Gb/s available PCIe bandwidth, limited by %s x%d link at %s (capable of %u.%03u Gb/s with %s x%d link)\n",
5668 			 bw_avail / 1000, bw_avail % 1000,
5669 			 PCIE_SPEED2STR(speed), width,
5670 			 limiting_dev ? pci_name(limiting_dev) : "<unknown>",
5671 			 bw_cap / 1000, bw_cap % 1000,
5672 			 PCIE_SPEED2STR(speed_cap), width_cap);
5673 }
5674 
5675 /**
5676  * pcie_print_link_status - Report the PCI device's link speed and width
5677  * @dev: PCI device to query
5678  *
5679  * Report the available bandwidth at the device.
5680  */
pcie_print_link_status(struct pci_dev * dev)5681 void pcie_print_link_status(struct pci_dev *dev)
5682 {
5683 	__pcie_print_link_status(dev, true);
5684 }
5685 EXPORT_SYMBOL(pcie_print_link_status);
5686 
5687 /**
5688  * pci_select_bars - Make BAR mask from the type of resource
5689  * @dev: the PCI device for which BAR mask is made
5690  * @flags: resource type mask to be selected
5691  *
5692  * This helper routine makes bar mask from the type of resource.
5693  */
pci_select_bars(struct pci_dev * dev,unsigned long flags)5694 int pci_select_bars(struct pci_dev *dev, unsigned long flags)
5695 {
5696 	int i, bars = 0;
5697 	for (i = 0; i < PCI_NUM_RESOURCES; i++)
5698 		if (pci_resource_flags(dev, i) & flags)
5699 			bars |= (1 << i);
5700 	return bars;
5701 }
5702 EXPORT_SYMBOL(pci_select_bars);
5703 
5704 /* Some architectures require additional programming to enable VGA */
5705 static arch_set_vga_state_t arch_set_vga_state;
5706 
pci_register_set_vga_state(arch_set_vga_state_t func)5707 void __init pci_register_set_vga_state(arch_set_vga_state_t func)
5708 {
5709 	arch_set_vga_state = func;	/* NULL disables */
5710 }
5711 
pci_set_vga_state_arch(struct pci_dev * dev,bool decode,unsigned int command_bits,u32 flags)5712 static int pci_set_vga_state_arch(struct pci_dev *dev, bool decode,
5713 				  unsigned int command_bits, u32 flags)
5714 {
5715 	if (arch_set_vga_state)
5716 		return arch_set_vga_state(dev, decode, command_bits,
5717 						flags);
5718 	return 0;
5719 }
5720 
5721 /**
5722  * pci_set_vga_state - set VGA decode state on device and parents if requested
5723  * @dev: the PCI device
5724  * @decode: true = enable decoding, false = disable decoding
5725  * @command_bits: PCI_COMMAND_IO and/or PCI_COMMAND_MEMORY
5726  * @flags: traverse ancestors and change bridges
5727  * CHANGE_BRIDGE_ONLY / CHANGE_BRIDGE
5728  */
pci_set_vga_state(struct pci_dev * dev,bool decode,unsigned int command_bits,u32 flags)5729 int pci_set_vga_state(struct pci_dev *dev, bool decode,
5730 		      unsigned int command_bits, u32 flags)
5731 {
5732 	struct pci_bus *bus;
5733 	struct pci_dev *bridge;
5734 	u16 cmd;
5735 	int rc;
5736 
5737 	WARN_ON((flags & PCI_VGA_STATE_CHANGE_DECODES) && (command_bits & ~(PCI_COMMAND_IO|PCI_COMMAND_MEMORY)));
5738 
5739 	/* ARCH specific VGA enables */
5740 	rc = pci_set_vga_state_arch(dev, decode, command_bits, flags);
5741 	if (rc)
5742 		return rc;
5743 
5744 	if (flags & PCI_VGA_STATE_CHANGE_DECODES) {
5745 		pci_read_config_word(dev, PCI_COMMAND, &cmd);
5746 		if (decode == true)
5747 			cmd |= command_bits;
5748 		else
5749 			cmd &= ~command_bits;
5750 		pci_write_config_word(dev, PCI_COMMAND, cmd);
5751 	}
5752 
5753 	if (!(flags & PCI_VGA_STATE_CHANGE_BRIDGE))
5754 		return 0;
5755 
5756 	bus = dev->bus;
5757 	while (bus) {
5758 		bridge = bus->self;
5759 		if (bridge) {
5760 			pci_read_config_word(bridge, PCI_BRIDGE_CONTROL,
5761 					     &cmd);
5762 			if (decode == true)
5763 				cmd |= PCI_BRIDGE_CTL_VGA;
5764 			else
5765 				cmd &= ~PCI_BRIDGE_CTL_VGA;
5766 			pci_write_config_word(bridge, PCI_BRIDGE_CONTROL,
5767 					      cmd);
5768 		}
5769 		bus = bus->parent;
5770 	}
5771 	return 0;
5772 }
5773 
5774 /**
5775  * pci_add_dma_alias - Add a DMA devfn alias for a device
5776  * @dev: the PCI device for which alias is added
5777  * @devfn: alias slot and function
5778  *
5779  * This helper encodes an 8-bit devfn as a bit number in dma_alias_mask
5780  * which is used to program permissible bus-devfn source addresses for DMA
5781  * requests in an IOMMU.  These aliases factor into IOMMU group creation
5782  * and are useful for devices generating DMA requests beyond or different
5783  * from their logical bus-devfn.  Examples include device quirks where the
5784  * device simply uses the wrong devfn, as well as non-transparent bridges
5785  * where the alias may be a proxy for devices in another domain.
5786  *
5787  * IOMMU group creation is performed during device discovery or addition,
5788  * prior to any potential DMA mapping and therefore prior to driver probing
5789  * (especially for userspace assigned devices where IOMMU group definition
5790  * cannot be left as a userspace activity).  DMA aliases should therefore
5791  * be configured via quirks, such as the PCI fixup header quirk.
5792  */
pci_add_dma_alias(struct pci_dev * dev,u8 devfn)5793 void pci_add_dma_alias(struct pci_dev *dev, u8 devfn)
5794 {
5795 	if (!dev->dma_alias_mask)
5796 		dev->dma_alias_mask = kcalloc(BITS_TO_LONGS(U8_MAX),
5797 					      sizeof(long), GFP_KERNEL);
5798 	if (!dev->dma_alias_mask) {
5799 		pci_warn(dev, "Unable to allocate DMA alias mask\n");
5800 		return;
5801 	}
5802 
5803 	set_bit(devfn, dev->dma_alias_mask);
5804 	pci_info(dev, "Enabling fixed DMA alias to %02x.%d\n",
5805 		 PCI_SLOT(devfn), PCI_FUNC(devfn));
5806 }
5807 
pci_devs_are_dma_aliases(struct pci_dev * dev1,struct pci_dev * dev2)5808 bool pci_devs_are_dma_aliases(struct pci_dev *dev1, struct pci_dev *dev2)
5809 {
5810 	return (dev1->dma_alias_mask &&
5811 		test_bit(dev2->devfn, dev1->dma_alias_mask)) ||
5812 	       (dev2->dma_alias_mask &&
5813 		test_bit(dev1->devfn, dev2->dma_alias_mask));
5814 }
5815 
pci_device_is_present(struct pci_dev * pdev)5816 bool pci_device_is_present(struct pci_dev *pdev)
5817 {
5818 	u32 v;
5819 
5820 	/* Check PF if pdev is a VF, since VF Vendor/Device IDs are 0xffff */
5821 	pdev = pci_physfn(pdev);
5822 	if (pci_dev_is_disconnected(pdev))
5823 		return false;
5824 	return pci_bus_read_dev_vendor_id(pdev->bus, pdev->devfn, &v, 0);
5825 }
5826 EXPORT_SYMBOL_GPL(pci_device_is_present);
5827 
pci_ignore_hotplug(struct pci_dev * dev)5828 void pci_ignore_hotplug(struct pci_dev *dev)
5829 {
5830 	struct pci_dev *bridge = dev->bus->self;
5831 
5832 	dev->ignore_hotplug = 1;
5833 	/* Propagate the "ignore hotplug" setting to the parent bridge. */
5834 	if (bridge)
5835 		bridge->ignore_hotplug = 1;
5836 }
5837 EXPORT_SYMBOL_GPL(pci_ignore_hotplug);
5838 
pcibios_default_alignment(void)5839 resource_size_t __weak pcibios_default_alignment(void)
5840 {
5841 	return 0;
5842 }
5843 
5844 #define RESOURCE_ALIGNMENT_PARAM_SIZE COMMAND_LINE_SIZE
5845 static char resource_alignment_param[RESOURCE_ALIGNMENT_PARAM_SIZE] = {0};
5846 static DEFINE_SPINLOCK(resource_alignment_lock);
5847 
5848 /**
5849  * pci_specified_resource_alignment - get resource alignment specified by user.
5850  * @dev: the PCI device to get
5851  * @resize: whether or not to change resources' size when reassigning alignment
5852  *
5853  * RETURNS: Resource alignment if it is specified.
5854  *          Zero if it is not specified.
5855  */
pci_specified_resource_alignment(struct pci_dev * dev,bool * resize)5856 static resource_size_t pci_specified_resource_alignment(struct pci_dev *dev,
5857 							bool *resize)
5858 {
5859 	int align_order, count;
5860 	resource_size_t align = pcibios_default_alignment();
5861 	const char *p;
5862 	int ret;
5863 
5864 	spin_lock(&resource_alignment_lock);
5865 	p = resource_alignment_param;
5866 	if (!*p && !align)
5867 		goto out;
5868 	if (pci_has_flag(PCI_PROBE_ONLY)) {
5869 		align = 0;
5870 		pr_info_once("PCI: Ignoring requested alignments (PCI_PROBE_ONLY)\n");
5871 		goto out;
5872 	}
5873 
5874 	while (*p) {
5875 		count = 0;
5876 		if (sscanf(p, "%d%n", &align_order, &count) == 1 &&
5877 		    p[count] == '@') {
5878 			p += count + 1;
5879 			if (align_order > 63) {
5880 				pr_err("PCI: Invalid requested alignment (order %d)\n",
5881 				       align_order);
5882 				align_order = PAGE_SHIFT;
5883 			}
5884 		} else {
5885 			align_order = PAGE_SHIFT;
5886 		}
5887 
5888 		ret = pci_dev_str_match(dev, p, &p);
5889 		if (ret == 1) {
5890 			*resize = true;
5891 			align = 1ULL << align_order;
5892 			break;
5893 		} else if (ret < 0) {
5894 			pr_err("PCI: Can't parse resource_alignment parameter: %s\n",
5895 			       p);
5896 			break;
5897 		}
5898 
5899 		if (*p != ';' && *p != ',') {
5900 			/* End of param or invalid format */
5901 			break;
5902 		}
5903 		p++;
5904 	}
5905 out:
5906 	spin_unlock(&resource_alignment_lock);
5907 	return align;
5908 }
5909 
pci_request_resource_alignment(struct pci_dev * dev,int bar,resource_size_t align,bool resize)5910 static void pci_request_resource_alignment(struct pci_dev *dev, int bar,
5911 					   resource_size_t align, bool resize)
5912 {
5913 	struct resource *r = &dev->resource[bar];
5914 	resource_size_t size;
5915 
5916 	if (!(r->flags & IORESOURCE_MEM))
5917 		return;
5918 
5919 	if (r->flags & IORESOURCE_PCI_FIXED) {
5920 		pci_info(dev, "BAR%d %pR: ignoring requested alignment %#llx\n",
5921 			 bar, r, (unsigned long long)align);
5922 		return;
5923 	}
5924 
5925 	size = resource_size(r);
5926 	if (size >= align)
5927 		return;
5928 
5929 	/*
5930 	 * Increase the alignment of the resource.  There are two ways we
5931 	 * can do this:
5932 	 *
5933 	 * 1) Increase the size of the resource.  BARs are aligned on their
5934 	 *    size, so when we reallocate space for this resource, we'll
5935 	 *    allocate it with the larger alignment.  This also prevents
5936 	 *    assignment of any other BARs inside the alignment region, so
5937 	 *    if we're requesting page alignment, this means no other BARs
5938 	 *    will share the page.
5939 	 *
5940 	 *    The disadvantage is that this makes the resource larger than
5941 	 *    the hardware BAR, which may break drivers that compute things
5942 	 *    based on the resource size, e.g., to find registers at a
5943 	 *    fixed offset before the end of the BAR.
5944 	 *
5945 	 * 2) Retain the resource size, but use IORESOURCE_STARTALIGN and
5946 	 *    set r->start to the desired alignment.  By itself this
5947 	 *    doesn't prevent other BARs being put inside the alignment
5948 	 *    region, but if we realign *every* resource of every device in
5949 	 *    the system, none of them will share an alignment region.
5950 	 *
5951 	 * When the user has requested alignment for only some devices via
5952 	 * the "pci=resource_alignment" argument, "resize" is true and we
5953 	 * use the first method.  Otherwise we assume we're aligning all
5954 	 * devices and we use the second.
5955 	 */
5956 
5957 	pci_info(dev, "BAR%d %pR: requesting alignment to %#llx\n",
5958 		 bar, r, (unsigned long long)align);
5959 
5960 	if (resize) {
5961 		r->start = 0;
5962 		r->end = align - 1;
5963 	} else {
5964 		r->flags &= ~IORESOURCE_SIZEALIGN;
5965 		r->flags |= IORESOURCE_STARTALIGN;
5966 		r->start = align;
5967 		r->end = r->start + size - 1;
5968 	}
5969 	r->flags |= IORESOURCE_UNSET;
5970 }
5971 
5972 /*
5973  * This function disables memory decoding and releases memory resources
5974  * of the device specified by kernel's boot parameter 'pci=resource_alignment='.
5975  * It also rounds up size to specified alignment.
5976  * Later on, the kernel will assign page-aligned memory resource back
5977  * to the device.
5978  */
pci_reassigndev_resource_alignment(struct pci_dev * dev)5979 void pci_reassigndev_resource_alignment(struct pci_dev *dev)
5980 {
5981 	int i;
5982 	struct resource *r;
5983 	resource_size_t align;
5984 	u16 command;
5985 	bool resize = false;
5986 
5987 	/*
5988 	 * VF BARs are read-only zero according to SR-IOV spec r1.1, sec
5989 	 * 3.4.1.11.  Their resources are allocated from the space
5990 	 * described by the VF BARx register in the PF's SR-IOV capability.
5991 	 * We can't influence their alignment here.
5992 	 */
5993 	if (dev->is_virtfn)
5994 		return;
5995 
5996 	/* check if specified PCI is target device to reassign */
5997 	align = pci_specified_resource_alignment(dev, &resize);
5998 	if (!align)
5999 		return;
6000 
6001 	if (dev->hdr_type == PCI_HEADER_TYPE_NORMAL &&
6002 	    (dev->class >> 8) == PCI_CLASS_BRIDGE_HOST) {
6003 		pci_warn(dev, "Can't reassign resources to host bridge\n");
6004 		return;
6005 	}
6006 
6007 	pci_read_config_word(dev, PCI_COMMAND, &command);
6008 	command &= ~PCI_COMMAND_MEMORY;
6009 	pci_write_config_word(dev, PCI_COMMAND, command);
6010 
6011 	for (i = 0; i <= PCI_ROM_RESOURCE; i++)
6012 		pci_request_resource_alignment(dev, i, align, resize);
6013 
6014 	/*
6015 	 * Need to disable bridge's resource window,
6016 	 * to enable the kernel to reassign new resource
6017 	 * window later on.
6018 	 */
6019 	if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE &&
6020 	    (dev->class >> 8) == PCI_CLASS_BRIDGE_PCI) {
6021 		for (i = PCI_BRIDGE_RESOURCES; i < PCI_NUM_RESOURCES; i++) {
6022 			r = &dev->resource[i];
6023 			if (!(r->flags & IORESOURCE_MEM))
6024 				continue;
6025 			r->flags |= IORESOURCE_UNSET;
6026 			r->end = resource_size(r) - 1;
6027 			r->start = 0;
6028 		}
6029 		pci_disable_bridge_window(dev);
6030 	}
6031 }
6032 
pci_set_resource_alignment_param(const char * buf,size_t count)6033 static ssize_t pci_set_resource_alignment_param(const char *buf, size_t count)
6034 {
6035 	if (count > RESOURCE_ALIGNMENT_PARAM_SIZE - 1)
6036 		count = RESOURCE_ALIGNMENT_PARAM_SIZE - 1;
6037 	spin_lock(&resource_alignment_lock);
6038 	strncpy(resource_alignment_param, buf, count);
6039 	resource_alignment_param[count] = '\0';
6040 	spin_unlock(&resource_alignment_lock);
6041 	return count;
6042 }
6043 
pci_get_resource_alignment_param(char * buf,size_t size)6044 static ssize_t pci_get_resource_alignment_param(char *buf, size_t size)
6045 {
6046 	size_t count;
6047 	spin_lock(&resource_alignment_lock);
6048 	count = snprintf(buf, size, "%s", resource_alignment_param);
6049 	spin_unlock(&resource_alignment_lock);
6050 	return count;
6051 }
6052 
pci_resource_alignment_show(struct bus_type * bus,char * buf)6053 static ssize_t pci_resource_alignment_show(struct bus_type *bus, char *buf)
6054 {
6055 	return pci_get_resource_alignment_param(buf, PAGE_SIZE);
6056 }
6057 
pci_resource_alignment_store(struct bus_type * bus,const char * buf,size_t count)6058 static ssize_t pci_resource_alignment_store(struct bus_type *bus,
6059 					const char *buf, size_t count)
6060 {
6061 	return pci_set_resource_alignment_param(buf, count);
6062 }
6063 
6064 static BUS_ATTR(resource_alignment, 0644, pci_resource_alignment_show,
6065 					pci_resource_alignment_store);
6066 
pci_resource_alignment_sysfs_init(void)6067 static int __init pci_resource_alignment_sysfs_init(void)
6068 {
6069 	return bus_create_file(&pci_bus_type,
6070 					&bus_attr_resource_alignment);
6071 }
6072 late_initcall(pci_resource_alignment_sysfs_init);
6073 
pci_no_domains(void)6074 static void pci_no_domains(void)
6075 {
6076 #ifdef CONFIG_PCI_DOMAINS
6077 	pci_domains_supported = 0;
6078 #endif
6079 }
6080 
6081 #ifdef CONFIG_PCI_DOMAINS_GENERIC
6082 static atomic_t __domain_nr = ATOMIC_INIT(-1);
6083 
pci_get_new_domain_nr(void)6084 static int pci_get_new_domain_nr(void)
6085 {
6086 	return atomic_inc_return(&__domain_nr);
6087 }
6088 
of_pci_bus_find_domain_nr(struct device * parent)6089 static int of_pci_bus_find_domain_nr(struct device *parent)
6090 {
6091 	static int use_dt_domains = -1;
6092 	int domain = -1;
6093 
6094 	if (parent)
6095 		domain = of_get_pci_domain_nr(parent->of_node);
6096 	/*
6097 	 * Check DT domain and use_dt_domains values.
6098 	 *
6099 	 * If DT domain property is valid (domain >= 0) and
6100 	 * use_dt_domains != 0, the DT assignment is valid since this means
6101 	 * we have not previously allocated a domain number by using
6102 	 * pci_get_new_domain_nr(); we should also update use_dt_domains to
6103 	 * 1, to indicate that we have just assigned a domain number from
6104 	 * DT.
6105 	 *
6106 	 * If DT domain property value is not valid (ie domain < 0), and we
6107 	 * have not previously assigned a domain number from DT
6108 	 * (use_dt_domains != 1) we should assign a domain number by
6109 	 * using the:
6110 	 *
6111 	 * pci_get_new_domain_nr()
6112 	 *
6113 	 * API and update the use_dt_domains value to keep track of method we
6114 	 * are using to assign domain numbers (use_dt_domains = 0).
6115 	 *
6116 	 * All other combinations imply we have a platform that is trying
6117 	 * to mix domain numbers obtained from DT and pci_get_new_domain_nr(),
6118 	 * which is a recipe for domain mishandling and it is prevented by
6119 	 * invalidating the domain value (domain = -1) and printing a
6120 	 * corresponding error.
6121 	 */
6122 	if (domain >= 0 && use_dt_domains) {
6123 		use_dt_domains = 1;
6124 	} else if (domain < 0 && use_dt_domains != 1) {
6125 		use_dt_domains = 0;
6126 		domain = pci_get_new_domain_nr();
6127 	} else {
6128 		if (parent)
6129 			pr_err("Node %pOF has ", parent->of_node);
6130 		pr_err("Inconsistent \"linux,pci-domain\" property in DT\n");
6131 		domain = -1;
6132 	}
6133 
6134 	return domain;
6135 }
6136 
pci_bus_find_domain_nr(struct pci_bus * bus,struct device * parent)6137 int pci_bus_find_domain_nr(struct pci_bus *bus, struct device *parent)
6138 {
6139 	return acpi_disabled ? of_pci_bus_find_domain_nr(parent) :
6140 			       acpi_pci_bus_find_domain_nr(bus);
6141 }
6142 #endif
6143 
6144 /**
6145  * pci_ext_cfg_avail - can we access extended PCI config space?
6146  *
6147  * Returns 1 if we can access PCI extended config space (offsets
6148  * greater than 0xff). This is the default implementation. Architecture
6149  * implementations can override this.
6150  */
pci_ext_cfg_avail(void)6151 int __weak pci_ext_cfg_avail(void)
6152 {
6153 	return 1;
6154 }
6155 
pci_fixup_cardbus(struct pci_bus * bus)6156 void __weak pci_fixup_cardbus(struct pci_bus *bus)
6157 {
6158 }
6159 EXPORT_SYMBOL(pci_fixup_cardbus);
6160 
pci_setup(char * str)6161 static int __init pci_setup(char *str)
6162 {
6163 	while (str) {
6164 		char *k = strchr(str, ',');
6165 		if (k)
6166 			*k++ = 0;
6167 		if (*str && (str = pcibios_setup(str)) && *str) {
6168 			if (!strcmp(str, "nomsi")) {
6169 				pci_no_msi();
6170 			} else if (!strncmp(str, "noats", 5)) {
6171 				pr_info("PCIe: ATS is disabled\n");
6172 				pcie_ats_disabled = true;
6173 			} else if (!strcmp(str, "noaer")) {
6174 				pci_no_aer();
6175 			} else if (!strcmp(str, "earlydump")) {
6176 				pci_early_dump = true;
6177 			} else if (!strncmp(str, "realloc=", 8)) {
6178 				pci_realloc_get_opt(str + 8);
6179 			} else if (!strncmp(str, "realloc", 7)) {
6180 				pci_realloc_get_opt("on");
6181 			} else if (!strcmp(str, "nodomains")) {
6182 				pci_no_domains();
6183 			} else if (!strncmp(str, "noari", 5)) {
6184 				pcie_ari_disabled = true;
6185 			} else if (!strncmp(str, "cbiosize=", 9)) {
6186 				pci_cardbus_io_size = memparse(str + 9, &str);
6187 			} else if (!strncmp(str, "cbmemsize=", 10)) {
6188 				pci_cardbus_mem_size = memparse(str + 10, &str);
6189 			} else if (!strncmp(str, "resource_alignment=", 19)) {
6190 				pci_set_resource_alignment_param(str + 19,
6191 							strlen(str + 19));
6192 			} else if (!strncmp(str, "ecrc=", 5)) {
6193 				pcie_ecrc_get_policy(str + 5);
6194 			} else if (!strncmp(str, "hpiosize=", 9)) {
6195 				pci_hotplug_io_size = memparse(str + 9, &str);
6196 			} else if (!strncmp(str, "hpmemsize=", 10)) {
6197 				pci_hotplug_mem_size = memparse(str + 10, &str);
6198 			} else if (!strncmp(str, "hpbussize=", 10)) {
6199 				pci_hotplug_bus_size =
6200 					simple_strtoul(str + 10, &str, 0);
6201 				if (pci_hotplug_bus_size > 0xff)
6202 					pci_hotplug_bus_size = DEFAULT_HOTPLUG_BUS_SIZE;
6203 			} else if (!strncmp(str, "pcie_bus_tune_off", 17)) {
6204 				pcie_bus_config = PCIE_BUS_TUNE_OFF;
6205 			} else if (!strncmp(str, "pcie_bus_safe", 13)) {
6206 				pcie_bus_config = PCIE_BUS_SAFE;
6207 			} else if (!strncmp(str, "pcie_bus_perf", 13)) {
6208 				pcie_bus_config = PCIE_BUS_PERFORMANCE;
6209 			} else if (!strncmp(str, "pcie_bus_peer2peer", 18)) {
6210 				pcie_bus_config = PCIE_BUS_PEER2PEER;
6211 			} else if (!strncmp(str, "pcie_scan_all", 13)) {
6212 				pci_add_flags(PCI_SCAN_ALL_PCIE_DEVS);
6213 			} else if (!strncmp(str, "disable_acs_redir=", 18)) {
6214 				disable_acs_redir_param = str + 18;
6215 			} else {
6216 				printk(KERN_ERR "PCI: Unknown option `%s'\n",
6217 						str);
6218 			}
6219 		}
6220 		str = k;
6221 	}
6222 	return 0;
6223 }
6224 early_param("pci", pci_setup);
6225 
6226 /*
6227  * 'disable_acs_redir_param' is initialized in pci_setup(), above, to point
6228  * to data in the __initdata section which will be freed after the init
6229  * sequence is complete. We can't allocate memory in pci_setup() because some
6230  * architectures do not have any memory allocation service available during
6231  * an early_param() call. So we allocate memory and copy the variable here
6232  * before the init section is freed.
6233  */
pci_realloc_setup_params(void)6234 static int __init pci_realloc_setup_params(void)
6235 {
6236 	disable_acs_redir_param = kstrdup(disable_acs_redir_param, GFP_KERNEL);
6237 
6238 	return 0;
6239 }
6240 pure_initcall(pci_realloc_setup_params);
6241