Home Explore Help
Sign In
universai
/
masyunis
1
0
Fork 0
Files Issues 0 Pull Requests 0 Wiki
Tree: 37107940ef
Branches Tags
masyunis
/
cmd
/
robots2policy
/
testdata
/
complex.yaml

complex.yaml 1.9 KB
History Raw

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071 
  1. - action: WEIGH
    2. 

  2.   expression: "true"
    3. 

  3.   name: robots-txt-policy-crawl-delay-1
    4. 

  4.   weight:
    5. 

  5.     adjust: 5
    6. 

  6. - action: CHALLENGE
    7. 

  7.   expression: path.startsWith("/admin/")
    8. 

  8.   name: robots-txt-policy-disallow-2
    9. 

  9. - action: CHALLENGE
    10. 

  10.   expression: path.startsWith("/private/")
    11. 

  11.   name: robots-txt-policy-disallow-3
    12. 

  12. - action: CHALLENGE
    13. 

  13.   expression: path.startsWith("/api/internal/")
    14. 

  14.   name: robots-txt-policy-disallow-4
    15. 

  15. - action: WEIGH
    16. 

  16.   expression: userAgent.contains("Googlebot")
    17. 

  17.   name: robots-txt-policy-crawl-delay-5
    18. 

  18.   weight:
    19. 

  19.     adjust: 5
    20. 

  20. - action: CHALLENGE
    21. 

  21.   expression:
    22. 

  22.     all:
    23. 

  23.       - userAgent.contains("Googlebot")
    24. 

  24.       - path.startsWith("/search/")
    25. 

  25.   name: robots-txt-policy-disallow-6
    26. 

  26. - action: WEIGH
    27. 

  27.   expression: userAgent.contains("Bingbot")
    28. 

  28.   name: robots-txt-policy-crawl-delay-7
    29. 

  29.   weight:
    30. 

  30.     adjust: 5
    31. 

  31. - action: CHALLENGE
    32. 

  32.   expression:
    33. 

  33.     all:
    34. 

  34.       - userAgent.contains("Bingbot")
    35. 

  35.       - path.startsWith("/search/")
    36. 

  36.   name: robots-txt-policy-disallow-8
    37. 

  37. - action: CHALLENGE
    38. 

  38.   expression:
    39. 

  39.     all:
    40. 

  40.       - userAgent.contains("Bingbot")
    41. 

  41.       - path.startsWith("/admin/")
    42. 

  42.   name: robots-txt-policy-disallow-9
    43. 

  43. - action: DENY
    44. 

  44.   expression: userAgent.contains("BadBot")
    45. 

  45.   name: robots-txt-policy-blacklist-10
    46. 

  46. - action: WEIGH
    47. 

  47.   expression: userAgent.contains("SeoBot")
    48. 

  48.   name: robots-txt-policy-crawl-delay-11
    49. 

  49.   weight:
    50. 

  50.     adjust: 5
    51. 

  51. - action: DENY
    52. 

  52.   expression: userAgent.contains("SeoBot")
    53. 

  53.   name: robots-txt-policy-blacklist-12
    54. 

  54. - action: CHALLENGE
    55. 

  55.   expression:
    56. 

  56.     all:
    57. 

  57.       - userAgent.contains("TestBot")
    58. 

  58.       - path.matches("^/.*/admin")
    59. 

  59.   name: robots-txt-policy-disallow-13
    60. 

  60. - action: CHALLENGE
    61. 

  61.   expression:
    62. 

  62.     all:
    63. 

  63.       - userAgent.contains("TestBot")
    64. 

  64.       - path.matches("^/temp.*\\.html")
    65. 

  65.   name: robots-txt-policy-disallow-14
    66. 

  66. - action: CHALLENGE
    67. 

  67.   expression:
    68. 

  68.     all:
    69. 

  69.       - userAgent.contains("TestBot")
    70. 

  70.       - path.matches("^/file.\\.log")
    71. 

  71.   name: robots-txt-policy-disallow-15
    72.