HTTP Headers 大全

复制import requests

# 检查 URL 返回的 X-Robots-Tag 和 Cache-Control 头
url = "https://example.com"
resp = requests.head(url, timeout=5)

x_robots = resp.headers.get("X-Robots-Tag", "(not set)")
cache_ctrl = resp.headers.get("Cache-Control", "(not set)")
canonical = resp.links.get("canonical", {}).get("url", "(not set)")

print(f"URL: {url}")
print(f"X-Robots-Tag: {x_robots}")
print(f"Cache-Control: {cache_ctrl}")
print(f"Canonical: {canonical}")
# 输出示例：
# URL: https://example.com
# X-Robots-Tag: noindex, nofollow
# Cache-Control: public, max-age=3600
# Canonical: https://example.com/

复制package main

import (
	"fmt"
	"net/http"
	"time"
)

func main() {
	url := "https://example.com"
	client := &http.Client{Timeout: 5 * time.Second}
	resp, err := client.Head(url)
	if err != nil {
		fmt.Println("请求失败:", err)
		return
	}
	defer resp.Body.Close()

	xRobots := resp.Header.Get("X-Robots-Tag")
	if xRobots == "" {
		xRobots = "(not set)"
	}
	cacheControl := resp.Header.Get("Cache-Control")
	if cacheControl == "" {
		cacheControl = "(not set)"
	}
	// Go 的 net/http 不直接解析 Link header，需手动提取
	linkHeader := resp.Header.Get("Link")
	canonical := "(not set)"
	if linkHeader != "" {
		// 简单提取 rel="canonical" 的 URL
		// 生产环境建议用正则或专用解析库
		canonical = "(see Link header: " + linkHeader + ")"
	}

	fmt.Printf("URL: %s\n", url)
	fmt.Printf("X-Robots-Tag: %s\n", xRobots)
	fmt.Printf("Cache-Control: %s\n", cacheControl)
	fmt.Printf("Canonical: %s\n", canonical)
}
// 输出示例：
// URL: https://example.com
// X-Robots-Tag: noindex, nofollow
// Cache-Control: public, max-age=3600
// Canonical: (see Link header: <https://example.com/>; rel="canonical")

复制// 使用 fetch 检查 HTTP 头（Node.js 18+ 或浏览器）
const url = 'https://example.com';

fetch(url, { method: 'HEAD' })
  .then(resp => {
    const xRobots = resp.headers.get('X-Robots-Tag') || '(not set)';
    const cacheControl = resp.headers.get('Cache-Control') || '(not set)';
    // 从 Link header 提取 canonical
    const linkHeader = resp.headers.get('Link') || '';
    let canonical = '(not set)';
    const match = linkHeader.match(/<([^>]+)>;\s*rel="canonical"/i);
    if (match) canonical = match[1];

    console.log(`URL: ${url}`);
    console.log(`X-Robots-Tag: ${xRobots}`);
    console.log(`Cache-Control: ${cacheControl}`);
    console.log(`Canonical: ${canonical}`);
  })
  .catch(err => console.error('请求失败:', err));
// 输出示例：
// URL: https://example.com
// X-Robots-Tag: noindex, nofollow
// Cache-Control: public, max-age=3600
// Canonical: https://example.com/